From 5fe231cc1942e48293d6e58cf148d06ea186b38d Mon Sep 17 00:00:00 2001 From: vividf Date: Thu, 2 Jul 2026 20:16:53 +0900 Subject: [PATCH 1/4] chore: fix centerpoint config Signed-off-by: vividf --- .../second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py b/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py index 5c55f45a6..4b6d02376 100644 --- a/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py +++ b/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py @@ -16,7 +16,7 @@ frame_pass_fail_config = dict( target_labels=_base_.class_names, # Matching thresholds per class (must align with `plane_distance_thresholds` used in evaluation) - matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0], + matching_threshold_list=[2.0] * len(_base_.class_names), confidence_threshold_list=None, ) From 9e4e0d8ccb6a627ee96c3b02a14bd5a3216cd1e9 Mon Sep 17 00:00:00 2001 From: vividf Date: Thu, 2 Jul 2026 20:20:46 +0900 Subject: [PATCH 2/4] chore: fix config Signed-off-by: vividf --- .../second_secfpn_8xb16_121m_base_amp_t4metric_v2.py | 2 +- .../second_secfpn_8xb16_121m_jpntaxi_base_amp_t4metric_v2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_base_amp_t4metric_v2.py b/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_base_amp_t4metric_v2.py index e2df1985a..75d5a051d 100644 --- a/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_base_amp_t4metric_v2.py +++ b/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_base_amp_t4metric_v2.py @@ -16,7 +16,7 @@ frame_pass_fail_config = dict( target_labels=_base_.class_names, # Matching thresholds per class (must align with `plane_distance_thresholds` used in evaluation) - matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0], + matching_threshold_list=[2.0] * len(_base_.class_names), confidence_threshold_list=None, ) diff --git a/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_jpntaxi_base_amp_t4metric_v2.py b/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_jpntaxi_base_amp_t4metric_v2.py index 7a38a41c6..341ae3fe3 100644 --- a/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_jpntaxi_base_amp_t4metric_v2.py +++ b/projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_jpntaxi_base_amp_t4metric_v2.py @@ -16,7 +16,7 @@ frame_pass_fail_config = dict( target_labels=_base_.class_names, # Matching thresholds per class (must align with `plane_distance_thresholds` used in evaluation) - matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0], + matching_threshold_list=[2.0] * len(_base_.class_names), confidence_threshold_list=None, ) From f049abd6346c95b0fd75d9e46a7bb5dc68e02753 Mon Sep 17 00:00:00 2001 From: vividf Date: Thu, 2 Jul 2026 16:16:48 +0900 Subject: [PATCH 3/4] feat(deployment): BEVFusion + CenterPoint deployment pipeline (no quantization) Deployment-only slice of the migrated deployment/ framework: - BEVFusion ONNX/TensorRT export + evaluation (single main_body and split sparse/dense), FP16/FP32, with sparse-encoder float shadow, SparseConv+BN fusion, and ImplicitGemm ReLU fusion for the sparse ONNX graph. - CenterPoint fp16/fp32 deploy configs and ONNX model loading. - Shared deployment framework edits (config schema/base, exporters, plugin loading) needed by the above. Contains NO quantization: the shared quantization framework, PTQ/QAT producers, INT8 deploy configs, the sparse INT8 ONNX transform, INT8/PTQ docs, and the C++ INT8 TensorRT plugin are all excluded. Signed-off-by: vividf --- deployment/config/base.py | 12 + deployment/config/schema.py | 9 +- .../evaluation/detection3d_evaluator.py | 130 +++ .../evaluation/point_detection_executor.py | 101 +++ deployment/export/exporters/configs.py | 3 + .../export/exporters/tensorrt_exporter.py | 5 + deployment/primitives/tensorrt_plugins.py | 129 +++ deployment/projects/bevfusion/README.md | 99 +++ deployment/projects/bevfusion/__init__.py | 19 + deployment/projects/bevfusion/cli.py | 16 + .../bevfusion/config/deploy_config.py | 163 ++++ .../deploy_config_split_fp16_opt_2_8.py | 198 +++++ ...OWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md | 184 ++++ ...ME_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md | 204 +++++ .../28_README_BEVFUSION_2_8_DEPLOYMENT.md | 127 +++ deployment/projects/bevfusion/docs/README.md | 12 + deployment/projects/bevfusion/entrypoint.py | 174 ++++ .../projects/bevfusion/evaluation/__init__.py | 0 .../bevfusion/evaluation/evaluator.py | 107 +++ .../projects/bevfusion/evaluation/executor.py | 58 ++ .../projects/bevfusion/export/__init__.py | 0 .../bevfusion/export/onnx_export_pipeline.py | 732 ++++++++++++++++ .../onnx_fuse_implicit_gemm_activation.py | 148 ++++ .../export/sparse_encoder_float_shadow.py | 295 +++++++ .../bevfusion/export/spconv_bn_fusion.py | 46 + .../export/tensorrt_export_pipeline.py | 136 +++ .../projects/bevfusion/inference/__init__.py | 0 .../inference/bevfusion_inference_pipeline.py | 310 +++++++ .../inference/onnx_inference_pipeline.py | 165 ++++ .../inference/pytorch_inference_pipeline.py | 234 +++++ .../inference/tensorrt_inference_pipeline.py | 808 ++++++++++++++++++ .../bevfusion/inference/trt_profiling.py | 188 ++++ deployment/projects/bevfusion/io/__init__.py | 0 .../projects/bevfusion/io/component_utils.py | 100 +++ .../projects/bevfusion/io/coors_contract.py | 29 + .../projects/bevfusion/io/data_loader.py | 82 ++ .../projects/bevfusion/io/model_loader.py | 65 ++ deployment/projects/bevfusion/runner.py | 92 ++ .../deploy_config_fp16_convnext_small.py | 166 ++++ .../deploy_config_fp16_convnext_standard.py | 162 ++++ .../config/deploy_config_fp16_resnet.py | 162 ++++ .../config/deploy_config_fp16_resnet_base.py | 162 ++++ .../config/deploy_config_fp16_second.py | 162 ++++ .../config/deploy_config_fp16_second_2_5.py | 162 ++++ .../config/deploy_config_fp16_second_2_6.py | 160 ++++ .../config/deploy_config_fp16_second_base.py | 162 ++++ .../config/deploy_config_fp16_vov57.py | 162 ++++ .../config/deploy_config_fp16_vov99.py | 162 ++++ .../centerpoint/config/deploy_config_fp32.py | 162 ++++ .../centerpoint/evaluation/evaluator.py | 191 +---- .../centerpoint/evaluation/executor.py | 108 +-- .../projects/centerpoint/io/model_loader.py | 4 + deployment/tests/test_centerpoint_configs.py | 67 ++ projects/BEVFusion/Dockerfile | 59 ++ projects/BEVFusion/README.md | 18 + projects/BEVFusion/bevfusion/bevfusion.py | 43 + ...econd_secfpn_30e_8xb16_j6gen2_base_120m.py | 3 +- .../BEVFusion/plugins/CMakeLists.standalone | 181 ++++ projects/BEVFusion/plugins/README.md | 95 ++ .../plugins/build_plugin_inside_container.sh | 279 ++++++ projects/SparseConvolution/sparse_conv.py | 43 +- .../SparseConvolution/sparse_functional.py | 63 +- 62 files changed, 7810 insertions(+), 308 deletions(-) create mode 100644 deployment/evaluation/detection3d_evaluator.py create mode 100644 deployment/evaluation/point_detection_executor.py create mode 100644 deployment/primitives/tensorrt_plugins.py create mode 100644 deployment/projects/bevfusion/README.md create mode 100644 deployment/projects/bevfusion/__init__.py create mode 100644 deployment/projects/bevfusion/cli.py create mode 100644 deployment/projects/bevfusion/config/deploy_config.py create mode 100644 deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py create mode 100644 deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md create mode 100644 deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md create mode 100644 deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md create mode 100644 deployment/projects/bevfusion/docs/README.md create mode 100644 deployment/projects/bevfusion/entrypoint.py create mode 100644 deployment/projects/bevfusion/evaluation/__init__.py create mode 100644 deployment/projects/bevfusion/evaluation/evaluator.py create mode 100644 deployment/projects/bevfusion/evaluation/executor.py create mode 100644 deployment/projects/bevfusion/export/__init__.py create mode 100644 deployment/projects/bevfusion/export/onnx_export_pipeline.py create mode 100644 deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py create mode 100644 deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py create mode 100644 deployment/projects/bevfusion/export/spconv_bn_fusion.py create mode 100644 deployment/projects/bevfusion/export/tensorrt_export_pipeline.py create mode 100644 deployment/projects/bevfusion/inference/__init__.py create mode 100644 deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py create mode 100644 deployment/projects/bevfusion/inference/onnx_inference_pipeline.py create mode 100644 deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py create mode 100644 deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py create mode 100644 deployment/projects/bevfusion/inference/trt_profiling.py create mode 100644 deployment/projects/bevfusion/io/__init__.py create mode 100644 deployment/projects/bevfusion/io/component_utils.py create mode 100644 deployment/projects/bevfusion/io/coors_contract.py create mode 100644 deployment/projects/bevfusion/io/data_loader.py create mode 100644 deployment/projects/bevfusion/io/model_loader.py create mode 100644 deployment/projects/bevfusion/runner.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py create mode 100644 deployment/projects/centerpoint/config/deploy_config_fp32.py create mode 100644 deployment/tests/test_centerpoint_configs.py create mode 100644 projects/BEVFusion/Dockerfile create mode 100644 projects/BEVFusion/plugins/CMakeLists.standalone create mode 100644 projects/BEVFusion/plugins/README.md create mode 100644 projects/BEVFusion/plugins/build_plugin_inside_container.sh diff --git a/deployment/config/base.py b/deployment/config/base.py index 1ad6d3cc7..886e5142e 100644 --- a/deployment/config/base.py +++ b/deployment/config/base.py @@ -143,6 +143,17 @@ def resolved_deploy_log_file(self) -> Optional[str]: work_dir = Path(self.export_config.work_dir).expanduser() return str((work_dir / log_path).resolve(strict=False)) + @property + def deploy_cfg(self) -> Config: + """Raw deploy-config object (read-only). + + Surfaces the original MMEngine ``Config`` so project-specific export pipelines can read + project-only keys (e.g. BEVFusion's ``fuse_spconv_bn``, ``bevfusion_merge``, + ``spconv_do_sort``, ``spconv_fuse_implicit_gemm_relu``) that the typed sections + intentionally do not model. + """ + return self._deploy_cfg + def get_verification_scenarios(self, export_mode: ExportMode) -> Tuple[VerificationScenario, ...]: """ Get verification scenarios for the given export mode. @@ -192,4 +203,5 @@ def get_tensorrt_settings(self, component_name: str) -> TensorRTExportConfig: precision_policy=self._tensorrt_config.precision_policy, max_workspace_size=self._tensorrt_config.max_workspace_size, model_input=model_input, + plugin_libraries=self._tensorrt_config.plugin_libraries, ) diff --git a/deployment/config/schema.py b/deployment/config/schema.py index 04627fdde..22401ae40 100644 --- a/deployment/config/schema.py +++ b/deployment/config/schema.py @@ -139,22 +139,29 @@ class TensorRTConfig: Configuration for TensorRT backend-specific settings. Uses config structure: - tensorrt_config = dict(precision_policy="auto", max_workspace_size=1<<30) + tensorrt_config = dict(precision_policy="auto", max_workspace_size=1<<30, + plugin_libraries=["/opt/plugins/libcustom.so"]) TensorRT profiles are defined in components.*.tensorrt_profile. Note: The deploy config key for this section is **`tensorrt_config`**. + + ``plugin_libraries`` lists custom TensorRT plugin ``.so`` paths to ``dlopen`` + before engine build/deserialize (e.g. the BEVFusion spconv ImplicitGemm plugin). Empty + by default, so projects that need no custom plugins (e.g. CenterPoint) are unaffected. """ precision_policy: PrecisionPolicy = PrecisionPolicy.AUTO max_workspace_size: int = DEFAULT_WORKSPACE_SIZE + plugin_libraries: Tuple[str, ...] = () @classmethod def from_dict(cls, config_dict: Mapping[str, Any]) -> TensorRTConfig: return cls( precision_policy=PrecisionPolicy.from_value(config_dict.get("precision_policy")), max_workspace_size=config_dict.get("max_workspace_size", DEFAULT_WORKSPACE_SIZE), + plugin_libraries=tuple(config_dict.get("plugin_libraries") or ()), ) diff --git a/deployment/evaluation/detection3d_evaluator.py b/deployment/evaluation/detection3d_evaluator.py new file mode 100644 index 000000000..d168b4b19 --- /dev/null +++ b/deployment/evaluation/detection3d_evaluator.py @@ -0,0 +1,130 @@ +"""Shared evaluator for point-cloud 3D detectors (CenterPoint, BEVFusion). + +Factors out the metrics hooks (prediction/GT parsing, metric accumulation, result building, +comparison summary) that the two projects' evaluators previously duplicated (~6 methods, near +verbatim). Subclasses override only ``print_results`` (backend-specific latency-breakdown +layout) and may reuse ``_log_latency_stats`` for the shared latency block. + +Note: this unifies the two former ``_build_results`` copies to the **stricter** CenterPoint +behavior — it validates the required summary keys and populates ``detailed_metrics`` with the +computed metrics. The old BEVFusion copy was lax (``.get(..., {})`` and empty detailed metrics); +that drift is intentionally removed here. +""" + +from __future__ import annotations + +import logging +from typing import Dict, List, Mapping + +import numpy as np +from mmengine.config import Config +from typing_extensions import override + +from deployment.evaluation.backend_executor import BackendExecutor +from deployment.evaluation.base_evaluator import BaseEvaluator, EvalResultDict +from deployment.metrics.detection_3d_metrics import Detection3DMetricsConfig, Detection3DMetricsInterface + +logger = logging.getLogger(__name__) + +_REQUIRED_SUMMARY_KEYS = ("mAP_by_mode", "mAPH_by_mode", "per_class_ap_by_mode") + + +class Detection3DEvaluator(BaseEvaluator): + """Evaluator base for 3D-detection deployment: metrics hooks shared across detectors. + + Backend execution (pipeline creation, input prep, device handling) is delegated to the + ``executor``; this class implements the task-generic metrics hooks. Subclasses provide only + ``print_results`` (the latency-breakdown layout differs per model). + + Args: + model_cfg: Model configuration; must have ``class_names``. + metrics_config: Configuration for 3D detection metrics (e.g. T4MetricV2). + executor: Backend execution primitives, shared with the verification runner. + """ + + def __init__( + self, + model_cfg: Config, + metrics_config: Detection3DMetricsConfig, + executor: BackendExecutor, + ) -> None: + if not hasattr(model_cfg, "class_names"): + raise ValueError("class_names must be provided via model_cfg.class_names.") + super().__init__( + metrics_interface=Detection3DMetricsInterface(metrics_config), + model_cfg=model_cfg, + executor=executor, + ) + + @override + def _parse_predictions(self, pipeline_output: object) -> List[Dict]: + """Return pipeline output as a list of prediction dicts (empty list if not a list).""" + return pipeline_output if isinstance(pipeline_output, list) else [] + + @override + def _parse_ground_truths(self, gt_data: Mapping[str, object]) -> List[Dict]: + """Convert ``gt_bboxes_3d`` / ``gt_labels_3d`` into ``[{"bbox_3d": [...], "label": int}]``.""" + if "gt_bboxes_3d" not in gt_data: + raise KeyError("gt_bboxes_3d not found in ground truth data.") + if "gt_labels_3d" not in gt_data: + raise KeyError("gt_labels_3d not found in ground truth data.") + + gt_bboxes_3d = np.asarray(gt_data["gt_bboxes_3d"], dtype=np.float32) + box_dim = gt_bboxes_3d.shape[-1] if gt_bboxes_3d.ndim > 1 else 7 + gt_bboxes_3d = gt_bboxes_3d.reshape(-1, box_dim) + gt_labels_3d = np.asarray(gt_data["gt_labels_3d"], dtype=np.int64).reshape(-1) + return [{"bbox_3d": gt_bboxes_3d[i].tolist(), "label": int(gt_labels_3d[i])} for i in range(len(gt_bboxes_3d))] + + @override + def _add_to_interface(self, predictions: List[Dict], ground_truths: List[Dict]) -> None: + """Add one frame of predictions and ground truths to the metrics interface.""" + self.metrics_interface.add_frame(predictions, ground_truths) + + @override + def _build_results( + self, + latencies: List[float], + latency_breakdowns: List[Dict[str, float]], + num_samples: int, + ) -> EvalResultDict: + """Build the result dict (mAP/mAPH, per-class AP, detailed metrics, latency, breakdown).""" + latency_stats = self.compute_latency_stats(latencies) + map_results = self.metrics_interface.compute_metrics() + summary_dict = self.metrics_interface.summary.to_dict() + missing = [k for k in _REQUIRED_SUMMARY_KEYS if k not in summary_dict] + if missing: + raise KeyError(f"Missing required metrics summary keys: {missing}") + + result: EvalResultDict = { + "mAP_by_mode": summary_dict["mAP_by_mode"], + "mAPH_by_mode": summary_dict["mAPH_by_mode"], + "per_class_ap_by_mode": summary_dict["per_class_ap_by_mode"], + "detailed_metrics": map_results, + "latency": latency_stats, + "num_samples": num_samples, + } + if latency_breakdowns: + result["latency_breakdown"] = self._compute_latency_breakdown(latency_breakdowns) + return result + + @override + def summarize_for_comparison(self, results: EvalResultDict) -> List[str]: + """Summarize mAP/mAPH per mode for the cross-backend comparison.""" + lines: List[str] = [] + for mode, map_value in (results.get("mAP_by_mode") or {}).items(): + lines.append(f" mAP ({mode}): {map_value:.4f}") + for mode, maph_value in (results.get("mAPH_by_mode") or {}).items(): + lines.append(f" mAPH ({mode}): {maph_value:.4f}") + lines.extend(super().summarize_for_comparison(results)) + return lines + + def _log_latency_stats(self, results: EvalResultDict) -> None: + """Log the shared latency-statistics block (mean/std/min/max/median).""" + latency_dict = results["latency"].to_dict() + logger.info("") + logger.info("Latency Statistics:") + logger.info(" Mean: %.2f ms", latency_dict["mean_ms"]) + logger.info(" Std: %.2f ms", latency_dict["std_ms"]) + logger.info(" Min: %.2f ms", latency_dict["min_ms"]) + logger.info(" Max: %.2f ms", latency_dict["max_ms"]) + logger.info(" Median: %.2f ms", latency_dict["median_ms"]) diff --git a/deployment/evaluation/point_detection_executor.py b/deployment/evaluation/point_detection_executor.py new file mode 100644 index 000000000..c1e32cd5b --- /dev/null +++ b/deployment/evaluation/point_detection_executor.py @@ -0,0 +1,101 @@ +"""Shared backend-execution primitives for point-cloud 3D detectors. + +``PointDetectionExecutor`` factors out the pipeline-construction and ``(points, metainfo)`` +input-prep that the CenterPoint and BEVFusion executors previously duplicated (~85% overlap). +Subclasses declare the three backend pipeline classes and, optionally, override +``get_output_names`` / ``_tensorrt_pipeline_kwargs``; everything else is shared. +""" + +from __future__ import annotations + +import logging +from typing import Any, Mapping, Optional, Type + +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.config.schema import ComponentsConfig +from deployment.evaluation.backend_executor import BackendExecutor +from deployment.evaluation.evaluator_types import InferenceInput, ModelSpec +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.io.base_data_loader import BaseDataLoader +from deployment.primitives.device import DeviceSpec + +logger = logging.getLogger(__name__) + + +class PointDetectionExecutor(BackendExecutor): + """Backend execution primitives shared by point-cloud 3D detectors (CenterPoint, BEVFusion). + + Subclasses set the three pipeline classes and (optionally) override ``get_output_names`` + and ``_tensorrt_pipeline_kwargs``. Pipeline construction and input prep are shared. + + Args: + components_cfg: Unified components configuration, forwarded to the ONNX/TensorRT + pipelines so they can resolve split vs merged artifacts. + """ + + #: Human-readable task name, used in log lines and error messages. + task_name: str = "point detector" + #: Backend pipeline classes; set as class attributes by subclasses. + pytorch_pipeline_cls: Optional[Type[BaseInferencePipeline]] = None + onnx_pipeline_cls: Optional[Type[BaseInferencePipeline]] = None + tensorrt_pipeline_cls: Optional[Type[BaseInferencePipeline]] = None + + def __init__(self, components_cfg: ComponentsConfig) -> None: + super().__init__() + self._components_cfg = components_cfg + + def _tensorrt_pipeline_kwargs(self) -> Mapping[str, Any]: + """Extra keyword args forwarded to the TensorRT pipeline (default: none). + + BEVFusion overrides this to pass its custom spconv ImplicitGemm ``plugin_libraries``. + """ + return {} + + @override + def create_pipeline(self, model_spec: ModelSpec, device: DeviceSpec) -> BaseInferencePipeline: + """Create a backend inference pipeline for ``model_spec.backend`` on ``device``.""" + backend = model_spec.backend + self._validate_backend(backend) + + if backend is Backend.PYTORCH: + logger.info("Creating %s PyTorch pipeline on %s", self.task_name, device) + return self.pytorch_pipeline_cls(self.pytorch_model, device=device) + + if backend is Backend.ONNX: + logger.info("Creating %s ONNX pipeline from %s on %s", self.task_name, model_spec.artifact.path, device) + return self.onnx_pipeline_cls( + self.pytorch_model, + onnx_dir=model_spec.artifact.path, + device=device, + components_cfg=self._components_cfg, + ) + + if backend is Backend.TENSORRT: + logger.info( + "Creating %s TensorRT pipeline from %s on %s", self.task_name, model_spec.artifact.path, device + ) + return self.tensorrt_pipeline_cls( + self.pytorch_model, + tensorrt_dir=model_spec.artifact.path, + device=device, + components_cfg=self._components_cfg, + **self._tensorrt_pipeline_kwargs(), + ) + + raise ValueError(f"Unsupported backend: {backend.value}") + + @override + def prepare_input( + self, + sample: Mapping[str, Any], + data_loader: BaseDataLoader, + device: DeviceSpec, + ) -> InferenceInput: + """Build InferenceInput from a sample containing ``points`` and ``metainfo``.""" + if "points" not in sample: + raise ValueError(f"Expected 'points' in sample. Got keys: {list(sample.keys())}") + if "metainfo" not in sample: + raise KeyError(f"Sample must contain 'metainfo' for {self.task_name} postprocess.") + return InferenceInput(data=sample["points"], metadata=sample["metainfo"]) diff --git a/deployment/export/exporters/configs.py b/deployment/export/exporters/configs.py index 20f6f839b..398200690 100644 --- a/deployment/export/exporters/configs.py +++ b/deployment/export/exporters/configs.py @@ -81,8 +81,11 @@ class TensorRTExportConfig: max_workspace_size: Workspace size in bytes. model_input: Per-input optimization-profile shapes. A single config already maps multiple named inputs via ``input_shapes``; None means no dynamic profile. + plugin_libraries: Custom TensorRT plugin ``.so`` paths to load before building + the engine (e.g. the BEVFusion spconv ImplicitGemm plugin). Empty by default. """ precision_policy: PrecisionPolicy = PrecisionPolicy.AUTO max_workspace_size: int = 1 << 30 model_input: Optional[TensorRTModelInputConfig] = None + plugin_libraries: Tuple[str, ...] = () diff --git a/deployment/export/exporters/tensorrt_exporter.py b/deployment/export/exporters/tensorrt_exporter.py index 6156b9996..387b43e0b 100644 --- a/deployment/export/exporters/tensorrt_exporter.py +++ b/deployment/export/exporters/tensorrt_exporter.py @@ -10,6 +10,7 @@ from deployment.config.enums import PrecisionPolicy from deployment.export.exporters.configs import TensorRTExportConfig from deployment.primitives.artifacts import Artifact +from deployment.primitives.tensorrt_plugins import load_tensorrt_plugin_libraries logger = logging.getLogger(__name__) @@ -57,6 +58,10 @@ def export( # Initialize TensorRT trt_logger = trt.Logger(trt.Logger.WARNING) + # Load any custom plugin .so libraries (e.g. the BEVFusion spconv ImplicitGemm plugin) + # before the built-in plugin init, so their creators are registered for engine build. + # No-op when plugin_libraries is empty (e.g. CenterPoint). + load_tensorrt_plugin_libraries(logger, self.config.plugin_libraries) trt.init_libnvinfer_plugins(trt_logger, "") builder = trt.Builder(trt_logger) diff --git a/deployment/primitives/tensorrt_plugins.py b/deployment/primitives/tensorrt_plugins.py new file mode 100644 index 000000000..f49dab9eb --- /dev/null +++ b/deployment/primitives/tensorrt_plugins.py @@ -0,0 +1,129 @@ +"""Utilities for loading TensorRT custom plugin libraries. + +Plugin paths are read only from deploy_config.tensorrt_config.plugin_libraries. +""" + +from __future__ import annotations + +import ctypes +import logging +import os +from typing import Iterable, List, Tuple + +_LOADED_PLUGIN_LIBS: set[str] = set() + + +def _normalize_libraries(plugin_libraries: Iterable[str]) -> List[str]: + normalized: List[str] = [] + seen: set[str] = set() + for path in plugin_libraries: + p = str(path).strip() + if not p: + continue + expanded = os.path.expandvars(os.path.expanduser(p)) + if expanded in seen: + continue + seen.add(expanded) + normalized.append(expanded) + return normalized + + +def _get_tensorrt_registries(): + """Best-effort fetch runtime and builder registries across TRT versions.""" + try: + import tensorrt as trt + except Exception: + return [] + + regs = [] + try: + runtime_registry = trt.get_plugin_registry() + if runtime_registry is not None: + regs.append(("runtime", runtime_registry)) + except Exception: + pass + + if hasattr(trt, "get_builder_plugin_registry"): + builder_registry = None + try: + builder_registry = trt.get_builder_plugin_registry() + except TypeError: + if hasattr(trt, "EngineCapability"): + try: + builder_registry = trt.get_builder_plugin_registry(trt.EngineCapability.STANDARD) + except Exception: + builder_registry = None + except Exception: + builder_registry = None + if builder_registry is not None: + regs.append(("builder", builder_registry)) + + uniq = [] + seen = set() + for name, reg in regs: + k = id(reg) + if k in seen: + continue + seen.add(k) + uniq.append((name, reg)) + return uniq + + +def load_tensorrt_plugin_libraries( + logger: logging.Logger, + plugin_libraries: Iterable[str], +) -> Tuple[str, ...]: + """Load custom TensorRT plugin libraries. + + Paths are taken only from deploy_config.tensorrt_config.plugin_libraries. + For TensorRT 10+ (Plugin V3), prefer registry-based loading + (`IPluginRegistry.load_library`) so creators are registered properly. + Fallback to `ctypes.CDLL(..., RTLD_GLOBAL)` for compatibility. + + Args: + logger: Logger used for informative diagnostics. + plugin_libraries: Plugin library paths from deploy config (e.g. tensorrt_config.plugin_libraries). + + Returns: + Tuple of loaded plugin library identifiers. + + Raises: + FileNotFoundError: If a configured path includes a slash but does not exist. + OSError: If dlopen fails for a provided library. + """ + resolved = _normalize_libraries(plugin_libraries) + loaded_now: List[str] = [] + + for library in resolved: + if library in _LOADED_PLUGIN_LIBS: + continue + + if "/" in library and not os.path.exists(library): + raise FileNotFoundError(f"TensorRT plugin library not found: {library}") + + # Try TensorRT registry loader first (required by many TRT10 V3 plugins). + loaded_with_registry = False + for reg_name, registry in _get_tensorrt_registries(): + if not hasattr(registry, "load_library"): + continue + try: + registry.load_library(library) + loaded_with_registry = True + logger.info("Loaded TensorRT plugin library via %s registry: %s", reg_name, library) + except Exception as exc: # pragma: no cover - best effort fallback path + logger.debug("%s registry load failed for %s: %s", reg_name, library, exc) + + # Always ensure symbols are globally visible for dependent shared objects. + if not loaded_with_registry: + ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) + logger.info("Loaded TensorRT plugin library via ctypes: %s", library) + else: + ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) + + _LOADED_PLUGIN_LIBS.add(library) + loaded_now.append(library) + + if not resolved: + logger.debug("No custom TensorRT plugin libraries configured. Set tensorrt_config.plugin_libraries.") + + return tuple(_LOADED_PLUGIN_LIBS) diff --git a/deployment/projects/bevfusion/README.md b/deployment/projects/bevfusion/README.md new file mode 100644 index 000000000..a7e56c106 --- /dev/null +++ b/deployment/projects/bevfusion/README.md @@ -0,0 +1,99 @@ +# BEVFusion Deployment — Architecture Guide + +> Orientation doc for engineers and agents working on BEVFusion deployment. It explains +> **how the BEVFusion project bundle is wired**: PyTorch checkpoint → ONNX (sparse + dense) → +> TensorRT → evaluation. Deep-dive notes live in [`docs/`](docs/README.md); this file is the map. + +For the framework-wide mental model, read [`deployment/docs/architecture.md`](../../docs/architecture.md) +first — BEVFusion is one *project bundle* that implements the stage contract described there. + +--- + +## 1. End-to-end flow + +```mermaid +flowchart TD + ckpt["FP32/FP16 checkpoint"] --> run["deployment.cli.main bevfusion"] + run --> load["model_loader
build model, load_checkpoint"] + load --> onnx["ONNX export
sparse.onnx + dense.onnx (or single main_body)"] + onnx --> trt["TensorRT engines
+ Autoware ImplicitGemm plugin (sparse)"] + trt --> eval["evaluate / verify
PyTorch vs ONNX vs TRT"] +``` + +Single entry point: + +```bash +python -m deployment.cli.main bevfusion --module main_body +``` + +--- + +## 2. BEVFusion project bundle (`deployment/projects/bevfusion/`) + +Mirrors the framework stage contract. Wiring: [`entrypoint.py:run`](entrypoint.py) builds config + +data loader + executor + evaluator, then [`runner.py:BEVFusionDeploymentRunner`](runner.py) +(a thin `BaseDeploymentRunner`) injects BEVFusion's ONNX/TensorRT export pipelines. + +| Stage | Directory | Key modules | +| --- | --- | --- | +| Config | [`config/`](config/) | `deploy_config.py` (single) + `deploy_config_split_fp16_opt_2_8.py` (split) (§4) | +| IO | [`io/`](io/) | [`model_loader.py`](io/model_loader.py) (build + `load_checkpoint`, optional sparse BN fuse), `data_loader.py`, `coors_contract.py` (voxel `[x,y,z]`→graph `[z,y,x]`), `component_utils.py` (split vs merged) | +| Export | [`export/`](export/) | [`onnx_export_pipeline.py`](export/onnx_export_pipeline.py) (sparse/dense/main_body wrappers, TopK fix, float shadow, ImplicitGemm ReLU fuse), `spconv_bn_fusion.py`, `sparse_encoder_float_shadow.py`, `onnx_fuse_implicit_gemm_activation.py`, `tensorrt_export_pipeline.py` | +| Inference | [`inference/`](inference/) | `pytorch_/onnx_/tensorrt_inference_pipeline.py` (all `preprocess→run→postprocess`) | +| Evaluation | [`evaluation/`](evaluation/) | `executor.py` (pipeline construction + output routing), `evaluator.py` (3D metrics + latency breakdown) | + +--- + +## 3. Sparse vs dense split + +BEVFusion (LiDAR) exports as two components so the dense tower can go to plain TensorRT while the +sparse tower uses the custom `ImplicitGemm` plugin: + +| | Sparse encoder (`pts_middle_encoder`) | Dense backbone/neck/head | +| --- | --- | --- | +| ONNX I/O | `voxels,coors,num_points → lidar_bev` | `lidar_bev → bbox_pred,score,label_pred` | +| ONNX op | `autoware::ImplicitGemm` (custom) | standard `Conv2d/ReLU/Add` | +| TensorRT | **custom plugin** `ImplicitGemm` (`libautoware_tensorrt_plugins.so`) | TRT-native | + +The sparse tower is traced through a **fused FP32 shadow encoder** +([`sparse_encoder_float_shadow.py`](export/sparse_encoder_float_shadow.py)) so BN can be folded +(`fuse_spconv_bn`) into a clean BN-free sparse ONNX without mutating the runtime model. Graph knobs: + +- `fuse_spconv_bn` — fold SparseConv+BN in `pts_middle_encoder` before export. +- `spconv_do_sort` — bake the pair-mask argsort attribute into the exported `ImplicitGemm` nodes. +- `spconv_fuse_implicit_gemm_relu` — fuse trailing Relu into `ImplicitGemm` (see + [`onnx_fuse_implicit_gemm_activation.py`](export/onnx_fuse_implicit_gemm_activation.py)). + +The `ImplicitGemm` TensorRT plugin (with the `do_sort` attribute) is built from an Autoware fork; +see [`projects/BEVFusion/plugins/README.md`](../../../projects/BEVFusion/plugins/README.md) and +[`projects/BEVFusion/Dockerfile`](../../../projects/BEVFusion/Dockerfile). + +--- + +## 4. Config variants (how to pick a mode) + +All configs are MMEngine files. + +| Config | Topology | Precision | +| --- | --- | --- | +| [`deploy_config.py`](config/deploy_config.py) | single `main_body` | FP32/FP16 | +| [`deploy_config_split_fp16_opt_2_8.py`](config/deploy_config_split_fp16_opt_2_8.py) | split sparse+dense | FP16 (optimized) | + +**Isolation tip:** to check whether the split/voxel/eval pipeline is healthy, keep the same +CLI/config/work_dir and point `checkpoint_path` at the FP32 `.pth`. If mAP is fine, the pipeline is +healthy and any regression is upstream (e.g. the checkpoint or model config). + +--- + +## 5. Where to go next + +- BEVFusion `coors` contract + Autoware/eval alignment → [`docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md`](docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md) +- `ScatterND → SECOND` ONNX trace differences → [`docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) +- BEVFusion 2.8.x deployment notes → [`docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md`](docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md) +- Doc index → [`docs/README.md`](docs/README.md) +- Shared framework internals → [`deployment/docs/architecture.md`](../../docs/architecture.md) + +> **Environment note:** ONNX/TensorRT export and evaluation run inside the BEVFusion deployment +> Docker image (see [`projects/BEVFusion/Dockerfile`](../../../projects/BEVFusion/Dockerfile)). The +> sparse `ImplicitGemm` TensorRT plugin `.so` must be built and present at the path in +> `tensorrt_config.plugin_libraries`. diff --git a/deployment/projects/bevfusion/__init__.py b/deployment/projects/bevfusion/__init__.py new file mode 100644 index 000000000..8d3942a29 --- /dev/null +++ b/deployment/projects/bevfusion/__init__.py @@ -0,0 +1,19 @@ +"""BEVFusion deployment bundle. + +Registers a ProjectAdapter into the global project_registry so the unified CLI can invoke it. +Supports LiDAR BEVFusion export to ONNX/TensorRT with evaluation and verification. +""" + +from __future__ import annotations + +from deployment.projects.bevfusion.cli import add_args +from deployment.projects.bevfusion.entrypoint import run +from deployment.projects.registry import ProjectAdapter, project_registry + +project_registry.register( + ProjectAdapter( + name="bevfusion", + add_args=add_args, + run=run, + ) +) diff --git a/deployment/projects/bevfusion/cli.py b/deployment/projects/bevfusion/cli.py new file mode 100644 index 000000000..f7e67f1a5 --- /dev/null +++ b/deployment/projects/bevfusion/cli.py @@ -0,0 +1,16 @@ +"""BEVFusion CLI extensions.""" + +from __future__ import annotations + +import argparse + + +def add_args(parser: argparse.ArgumentParser) -> None: + """Register BEVFusion-specific CLI flags onto a project subparser.""" + parser.add_argument( + "--module", + type=str, + default="main_body", + choices=["main_body", "image_backbone", "camera_bev_only"], + help="Module to export (default: main_body)", + ) diff --git a/deployment/projects/bevfusion/config/deploy_config.py b/deployment/projects/bevfusion/config/deploy_config.py new file mode 100644 index 000000000..a39e447bf --- /dev/null +++ b/deployment/projects/bevfusion/config/deploy_config.py @@ -0,0 +1,163 @@ +""" +BEVFusion Deployment Configuration + +Example deploy config for BEVFusion LiDAR-only (main_body module). +Adapt checkpoint_path, info_file, and shape profiles to your model. +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/bevfusion/bevfusion_epoch_30.pth" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="onnx", + work_dir="work_dirs/bevfusion_deployment", + onnx_path=None, +) + +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Component Configuration +# +# BEVFusion exports a single ONNX model (main_body) that takes +# voxels/coors/num_points_per_voxel and outputs bbox_pred/score/label_pred. +# ============================================================================ +components = dict( + bevfusion_main_body=dict( + onnx_file="bevfusion_lidar.onnx", + engine_file="bevfusion_lidar.engine", + io=dict( + inputs=[ + dict(name="voxels", dtype="float32"), + dict(name="coors", dtype="int32"), + dict(name="num_points_per_voxel", dtype="int32"), + ], + outputs=[ + dict(name="bbox_pred", dtype="float32"), + dict(name="score", dtype="float32"), + dict(name="label_pred", dtype="int64"), + ], + dynamic_axes={ + "voxels": {0: "voxels_num"}, + "coors": {0: "voxels_num"}, + "num_points_per_voxel": {0: "voxels_num"}, + }, + ), + tensorrt_profile=dict( + voxels=dict( + min_shape=[1, 10, 5], + opt_shape=[64000, 10, 5], + max_shape=[256000, 10, 5], + ), + coors=dict( + min_shape=[1, 3], + opt_shape=[64000, 3], + max_shape=[256000, 3], + ), + num_points_per_voxel=dict( + min_shape=[1], + opt_shape=[64000], + max_shape=[256000], + ), + ), + ), +) + +# ============================================================================ +# Runtime I/O settings +# ============================================================================ +runtime_io = dict( + info_file="info/t4dataset_j6gen2_base_infos_test.pkl", + sample_idx=0, +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=17, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +# BEVFusion ONNX uses ImplicitGemm / GetIndicePairsImplicitGemm (spconv). You must +# provide the plugin .so and list it here, or TensorRT export will fail with +# "Plugin not found". See: projects/BEVFusion/plugins/README.md +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=1 << 32, + # Optional: enable FP16 at build time via policy_flags=dict(FP16=True). + # Set this after placing libautoware_tensorrt_plugins.so in the image (e.g. under + # /opt/plugins/). Alternatively set env DEPLOY_TENSORRT_PLUGIN_LIBS. + plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"], + # plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"] +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=5, + num_warmup=2, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict[str, bool | str]( + enabled=False, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cuda", test_backend="onnx", test_device="cuda"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cuda", test_backend="onnx", test_device="cuda"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py b/deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py new file mode 100644 index 000000000..3d7167ac8 --- /dev/null +++ b/deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py @@ -0,0 +1,198 @@ +""" +BEVFusion deploy config — **split ONNX / TensorRT (route 1)** + +Use this instead of ``deploy_config.py`` when you want: + - ``bevfusion_sparse.onnx`` / ``.engine`` — voxelization stays outside; graph is + ``pts_middle_encoder`` only (spconv / plugins / libspconv). + - ``bevfusion_dense.onnx`` / ``.engine`` — ``pts_backbone`` + ``pts_neck`` + ``bbox_head`` + (+ same postprocess as single-file export). Suitable for plain TensorRT without spconv ops. + +**Requirements** + - LiDAR-only model: ``fusion_layer is None`` and ``img_backbone is None``. + + - Set ``bevfusion_dense.tensorrt_profile.lidar_bev`` **H,W** to ``grid_size[0:2] // out_size_factor`` + (e.g. 1440/8 → **180×180**). Do **not** use a wide H/W range: ``bbox_head`` uses fixed ``bev_pos`` + and heatmap length ``H*W``; TRT profiles like 32×32 or 2048×2048 break ``Reshape``/``Gather`` + consistency and yield garbage mAP. + - Adjust channel **C** (default ``256``) to match ``pts_backbone.in_channels`` / sparse tower output. + +CLI:: + + python -m deployment.cli.main bevfusion \\ + deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py \\ + +""" + +spconv_do_sort = False + +# ============================================================================ +# Sparse ONNX postprocess (FP): fuse ImplicitGemm with trailing Relu/Add(const)+Relu. +# ---------------------------------------------------------------------------- +# Applied automatically by deployment/projects/bevfusion/export/onnx_export_pipeline.py +# after exporting ``bevfusion_sparse.onnx``. +# - True : bake activation into ImplicitGemm (act_type / optional 6th bias input) +# - False : keep explicit Relu/Add nodes +# ============================================================================ +spconv_fuse_implicit_gemm_relu = True + +# Fuse SparseConv + BN in ``pts_middle_encoder`` before ONNX export (eval-mode Conv-BN fold). +# Produces a BN-free sparse subgraph in the exported ONNX. +fuse_spconv_bn = True + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/bevfusion/bevfusion_2_8/best_epoch_25.pth" +# checkpoint_path = "vivid/bench_comparison/bevfusion_2_7/best_epoch_28.pth" + + +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +export = dict( + mode="both", + work_dir="work_dirs/bevfusion_deployment_2_8", + onnx_path="work_dirs/bevfusion_deployment_2_8/onnx", +) + + +# Optional: keep split component definitions for debugging, but export/eval as one main body. +# - False: split sparse+dense ONNX/engine (default) +# - True : one ONNX + one engine + one backend pipeline +bevfusion_merge = dict( + enabled=True, + onnx_file="bevfusion_lidar_fp16_opt.onnx", + engine_file="bevfusion_lidar_fp16_opt.engine", +) + + +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Split components (must keep keys ``bevfusion_sparse`` + ``bevfusion_dense``) +# ============================================================================ +components = dict( + bevfusion_sparse=dict( + onnx_file="bevfusion_sparse.onnx", + engine_file="bevfusion_sparse.engine", + io=dict( + inputs=[ + dict(name="voxels", dtype="float32"), + dict(name="coors", dtype="int32"), + dict(name="num_points_per_voxel", dtype="int32"), + ], + outputs=[ + dict(name="lidar_bev", dtype="float32"), + ], + dynamic_axes={ + "voxels": {0: "voxels_num"}, + "coors": {0: "voxels_num"}, + "num_points_per_voxel": {0: "voxels_num"}, + "lidar_bev": {0: "batch", 2: "bev_h", 3: "bev_w"}, + }, + ), + tensorrt_profile=dict( + voxels=dict( + min_shape=[1, 32, 5], + opt_shape=[64000, 32, 5], + max_shape=[256000, 32, 5], + ), + coors=dict( + min_shape=[1, 3], + opt_shape=[64000, 3], + max_shape=[256000, 3], + ), + num_points_per_voxel=dict( + min_shape=[1], + opt_shape=[64000], + max_shape=[256000], + ), + ), + ), + bevfusion_dense=dict( + onnx_file="bevfusion_dense.onnx", + engine_file="bevfusion_dense.engine", + io=dict( + inputs=[ + dict(name="lidar_bev", dtype="float32"), + ], + outputs=[ + dict(name="bbox_pred", dtype="float32"), + dict(name="score", dtype="float32"), + dict(name="label_pred", dtype="int64"), + ], + # Spatial dims must stay at head BEV resolution; see module docstring. + dynamic_axes={ + "lidar_bev": {0: "batch"}, + }, + ), + # H,W fixed to head grid (grid_size // out_size_factor). Widen only batch dim if needed. + tensorrt_profile=dict( + lidar_bev=dict( + min_shape=[1, 256, 180, 180], + opt_shape=[1, 256, 180, 180], + max_shape=[1, 256, 180, 180], + ), + ), + ), +) + +runtime_io = dict( + info_file="info/t4dataset_j6gen2_base_infos_test.pkl", + sample_idx=0, +) + +onnx_config = dict( + # BEVFusion 2.8.x exports at opset 18 (matches projects/BEVFusion/configs/deploy/*_tensorrt_dynamic.py). + opset_version=18, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=1 << 32, + plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"], +) + +evaluation = dict( + enabled=True, + num_samples=5, + num_warmup=2, + verbose=True, + backends=dict( + pytorch=dict( + enabled=False, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +verification = dict( + enabled=False, + tolerance=1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[], + onnx=[], + trt=[], + none=[], + ), +) diff --git a/deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md b/deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md new file mode 100644 index 000000000..17eec1fe3 --- /dev/null +++ b/deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md @@ -0,0 +1,184 @@ +# 25: BEVFusion `coors` 契約對齊 Autoware 與 Evaluation 修正 + +本文件記錄一個關鍵修正：讓 deployment framework 的 ONNX/TRT evaluation，對齊舊版（Autoware 相容）BEVFusion main-body ONNX 的 `coors` 契約，避免出現「PyTorch 正常、TRT mAP 接近 0」的錯誤對照結果。 + +--- + +## 1. 問題現象 + +同一個 checkpoint 與資料集，出現以下不一致： + +- `original` ONNX：PyTorch mAP 正常，TRT mAP 幾乎 0 +- `opt/new` ONNX：PyTorch mAP 正常，TRT mAP 正常 +- 但 `opt/new` 在實際 Autoware 路徑上無有效輸出（mAP~0） + +這代表 framework evaluation 與 Autoware 真實執行路徑存在契約不一致。 + +--- + +## 2. 根因：`coors` 的順序契約不一致 + +舊版 deploy（mmdeploy patch）在資料前處理與圖內 wrapper 有明確註解： + +```43:50:projects/BEVFusion/deploy/voxel_detection.py +# The original code/model uses [batch, x, y, z] +# but the data_preprocessor used here uses [batch, z, y, x] +# Since this is outside the graph we format it as [z, y, x] +# and convert it to [batch, x, y, z] inside the graph +``` + +對應到舊 wrapper 的圖內邏輯： + +```44:49:projects/BEVFusion/deploy/containers.py +if coors.shape[1] == 3: + coors = coors.flip(dims=[-1]).contiguous() # [z,y,x] -> [x,y,z] + batch_coors = torch.zeros(num_points, 1).to(coors.device) + coors = torch.cat([batch_coors, coors], dim=1).contiguous() +``` + +也就是： + +- graph 外輸入 `coors`：`[z, y, x]`（無 batch） +- graph 內先 flip 成 `[x, y, z]`，再 prepend batch + +若 evaluation runtime 沒遵守同一契約，就會把座標送錯位，TRT 精度會崩。 + +--- + +## 3. 這次修正做了什麼 + +### 3.1 ONNX export：明確保留 legacy contract + +在新 framework 的 ONNX wrapper 中加入與舊版一致的正規化。 + +- 檔案：`deployment/projects/bevfusion/export/onnx_export_pipeline.py` +- 函式：`_normalize_sparse_coors_for_autoware()` +- 行為：`[N,3] coors` 先 `flip(-1)`，再補 batch 欄位 + +目的：讓新匯出的 ONNX 與舊版 Autoware 相容圖在 `coors` 契約上等價。 + +### 3.2 ONNX / TRT runtime pipeline：餵入前對齊同一契約 + +在 framework inference backend（非 metric/evaluator）端加入同樣契約對齊。 + +- `deployment/projects/bevfusion/pipelines/onnx.py` +- `deployment/projects/bevfusion/pipelines/tensorrt.py` +- 函式：`_normalize_coors_for_legacy_main_body_contract()` +- 行為：餵 backend 前，對 `[N,3]` coors 做 `flip(-1)` + +目的：讓 runtime 輸入與 legacy main-body ONNX 的圖內假設一致。 + +### 3.3 沒有在 evaluator 做 flip（刻意） + +`evaluator` 只應負責 metrics，不應修改模型輸入語義。 +flip 必須放在模型邊界（export wrapper / backend preprocess），且只做一次。 + +### 3.4 `5132ce649385e0a32ddc9b35821b29e5e69eed66` 的具體改動 + +該 commit（`chore: temp fix`）實際做了以下 5 項： + +1. 新增 `deployment/projects/bevfusion/io/coors_contract.py` + - 統一定義兩個轉換函式： + - `voxel_indices_xyz_to_graph_input_zyx()` + - `graph_input_zyx_to_model_indices_xyz()` +2. 修改 `deployment/projects/bevfusion/export/onnx_export_pipeline.py` + - `_normalize_sparse_coors_for_autoware()` 改為呼叫 `coors_contract`，而非散落的 `flip` + - `_voxelize()` 明確把 voxel layer 輸出的 `[x,y,z]` 轉成 graph input `[z,y,x]` +3. 修改 `deployment/projects/bevfusion/pipelines/onnx.py` + - backend 餵入前改用 `coors_contract` 做 `xyz -> zyx` +4. 修改 `deployment/projects/bevfusion/pipelines/tensorrt.py` + - backend 餵入前改用 `coors_contract` 做 `xyz -> zyx` +5. 新增測試型 config + - `deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_on_board_test.py` + +--- + +## 4. `5132ce64` 是否合理（判斷） + +### 4.1 合理的部分（建議保留） + +- **把契約集中到單一模組 `coors_contract.py` 是正確方向** + 可避免 export / ORT / TRT 各自手寫 `flip`，降低再度漂移風險。 +- **在 `_voxelize()` 明確標註與轉換 `xyz -> zyx` 是關鍵修正** + 這一步把「voxelization 真實輸出順序」與「legacy graph input 契約」橋接清楚，對消除 `PyTorch OK / TRT ~0` 很重要。 +- **ORT/TRT backend 同步使用同一轉換** + 保證 runtime 行為一致，不會某 backend 正常、某 backend 失真。 + +### 4.2 需要注意的部分（不一定錯，但要標記） + +- commit message 是 `temp fix`，但內容其實是**正式契約修正**；建議後續用更明確訊息（例如 `fix: align BEVFusion coors contract with legacy autoware main_body`）。 +- 新增的 `*_on_board_test.py` 是測試配置，建議在文件中註明用途，避免被誤當 production baseline。 + +### 4.3 總結判斷 + +`5132ce64` 對核心問題是**合理且必要**的： +它不是「為了拉高單次指標的 overfit hack」，而是把 `coors` 契約明文化並在 export/runtime 一致落地。 + +--- + +## 5. 為什麼這樣改後 `original` 和 `new export` 都能正常 + +修正後，三者契約一致： + +1. 資料前處理輸出 `coors`（framework path） +2. backend 餵入前的 `coors` 正規化 +3. ONNX 圖內對 `coors` 的假設（legacy style） + +當這三件事一致，就不會再出現 `PyTorch OK / TRT ~0` 的錯位結果。 + +--- + +## 6. 這件事跟 ROS `x/y/z` 有沒有關係？ + +### 5.1 有關的部分 + +- ROS/Autoware 的座標系通常使用右手系（例如 `base_link`：`x` 前、`y` 左、`z` 上）。 +- `coors` 的欄位命名（`x,y,z` 或 `z,y,x`）在語意上會讓人聯想到座標軸方向。 + +### 5.2 無直接關聯的部分（本次核心） + +本次 bug 的核心不是 ROS frame 本身，而是**稀疏體素索引張量的欄位順序契約**： + +- 這是 tensor/算子接口契約問題（graph 外是什麼順序，graph 內期望什麼順序） +- 不等同於「把物理世界座標軸定義改掉」 + +換句話說： + +- ROS frame 定義決定「物理座標如何表示」 +- 本次修正決定「已經體素化後的 index tensor 欄位如何對齊算子契約」 + +兩者相關，但不是同一層問題。 + +--- + +## 7. 如何驗證契約是否一致 + +### 6.1 ONNX 檢查（legacy flip 是否存在） + +可檢查 `coors` 路徑是否有 reverse-slice（`steps=-1`）或等價 flip。 + +### 6.2 Backend 檢查（runtime 是否做對齊） + +檢查 ONNX/TRT pipeline 在 `run_bevfusion()` 中，`coors` 是否先經過 legacy 正規化再轉 numpy。 + +### 6.3 指標檢查 + +同資料、同 checkpoint： + +- PyTorch 應維持正常（作為 reference） +- TRT 與 ONNX 不應再是接近 0 的異常值 + +--- + +## 8. 設計原則（後續維護） + +1. **單一路徑契約**：不要同時維護「evaluation 專用語義」與「Autoware 實車語義」兩套邏輯。 +2. **翻轉只在模型邊界做**：不得在 evaluator 裡補救。 +3. **文件化契約**：`coors` 的輸入順序、圖內期望順序、batch 拼接位置必須固定記錄。 + +--- + +## 9. 結論 + +本次修正不是單純「讓分數變好」，而是把 evaluation 對齊回 legacy Autoware-compatible ONNX contract。 +核心是 `coors` 欄位順序契約一致化；ROS `x/y/z` 是背景語意，但本次直接修復點是 tensor index order 對齊，而非改動物理座標系定義。 diff --git a/deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md b/deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md new file mode 100644 index 000000000..9bceb4af1 --- /dev/null +++ b/deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md @@ -0,0 +1,204 @@ +# 26: 為什麼 `ScatterND -> SECOND` 在不同 ONNX 不是一模一樣 + +本文件說明一個常見疑問： + +- `original`（舊 main-body 單圖）ONNX 在 `ScatterND` 後常看到 + `Transpose -> Shape -> Gather -> Unsqueeze -> ...` +- `bevfusion_deployment_2_7_fp16_opt_merged_flip_no_opt`（split export 再 merge）ONNX + 常只看到短鏈（例如 `Transpose -> Reshape -> Conv`） + +這不是單純「誰對誰錯」，而是 **trace 邊界** 與 **圖組合方式** 不同造成的 ONNX 表達差異。 + +--- + +## 1. `ScatterND` 後面那串在做什麼 + +在 sparse encoder，`conv_out.dense()` 之後要把 5D 張量整理成 BEV 4D： + +```20:36:projects/BEVFusion/bevfusion/sparse_encoder.py +x = out_tensor.dense() # (N, C, H, W, D) +t = x.permute(0, 1, 4, 2, 3).contiguous() # (N, C, D, H, W) +return t.view(n, c * int(d), int(h), int(w)) +``` + +PyTorch 的 `permute + view` 在 ONNX 常被展開成： + +- `Transpose`（維度重排） +- `Shape/Gather`（抽出 N/C/H/W/D） +- `Unsqueeze/Concat`（組新 shape 向量） +- `Reshape` + +所以在 `original` 裡看到 `Transpose -> Shape -> Gather -> Unsqueeze` 是合理的。 + +--- + +## 2. 為什麼 split trace 後看起來「少很多」 + +### 2.1 匯出邊界不同（核心） + +`original` 是「單次 trace 全主圖」： +`voxels/coors -> sparse -> dense backbone/neck/head` 一次畫完。 + +split export 是兩次 trace： + +1. sparse 子圖：`voxels/coors -> lidar_bev` +2. dense 子圖：`lidar_bev -> backbone/neck/head` + +對應程式： + +- sparse wrapper：`BEVFusionSparseWrapper` +- dense wrapper：`BEVFusionDenseWrapper` +- merge：`onnx.compose.merge_models` + +```223:231:/home/yihsiangfang/ml_workspace/AWML/deployment/projects/bevfusion/export/onnx_export_pipeline.py +def _export_split(...): + """Export ``bevfusion_sparse.onnx`` and ``bevfusion_dense.onnx``.""" +``` + +### 2.1.1 分開 trace vs 一起 trace：各自用什麼方式 + +- **一起 trace（single-file）** + - 入口：`BEVFusionONNXExportPipeline.export()` 的 single-file 路徑 + - 包裝：`BEVFusionMainBodyWrapper` + - 方法：呼叫一次 `self._export_to_onnx(...)`，直接把 + `voxels/coors/num_points_per_voxel -> bbox_pred/score/label` 全流程 trace 成同一張 ONNX + - 特性：shape 推導鏈通常完整保留在同一張圖內（較常看到 `Shape/Gather/Unsqueeze`） + +- **分開 trace（split export）** + - 入口：`BEVFusionONNXExportPipeline._export_split()` + - 包裝與方法： + 1. sparse 段：`BEVFusionSparseWrapper` + `self._export_to_onnx(..., wrapper="sparse")` + 2. dense 段：先用 sparse wrapper 跑出 `lidar_bev`，再用 `self._export_dense_to_onnx(...)` 匯出 dense + 3. 合併段：`onnx.compose.add_prefix + onnx.compose.merge_models + cleanup().toposort()` + - 特性：merge 後常看到更短的 shape 鏈，因為跨子圖的中介 shape plumbing 容易被折疊 + +### 2.2 graph compose + cleanup 會消掉中介 shape plumbing + +split merge 之後會做 `cleanup().toposort()`，一些跨段的中介 shape 節點會被折疊或改寫： + +```488:490:/home/yihsiangfang/ml_workspace/AWML/deployment/projects/bevfusion/export/onnx_export_pipeline.py +merged_graph.cleanup().toposort() +onnx.save_model(gs.export_onnx(merged_graph), str(merged_path)) +``` + +因此你在 merged 圖常看到更短鏈（例如 `Transpose -> Reshape -> Conv`），而不是單圖 trace 時那種長 shape plumbing。 + +--- + +## 3. 為什麼「分開 trace」會比較少 `Shape/Gather/Unsqueeze` + +### 3.1 這串節點在做什麼 + +`Shape -> Gather -> Unsqueeze -> Concat` 是 ONNX 表達「**在 runtime 才讀取某個維度的值**」的方式。 + +在 `_conv_out_to_bev`（`sparse_encoder.py`）： + +```python +x = out_tensor.dense() # (N, C, H, W, D) +n = int(x.shape[0]) # ← 這行是關鍵：讀取 batch size +t = x.permute(0, 1, 4, 2, 3) +return t.view(n, c * int(d), int(h), int(w)) +``` + +當 `n` 是「要到 runtime 才知道的值」，ONNX 沒辦法寫死 `Reshape(t, [n, 256, 180, 180])`， +必須展開成完整的 shape 讀取鏈： + +``` +Shape(x) → [N, C, H, W, D] + ↓ +Gather(index=0) → N + ↓ +Unsqueeze → [N] + ↓ +Concat([N], [256], [180], [180]) → shape vector [N, 256, 180, 180] + ↓ +Reshape(t, shape_vector) +``` + +### 3.2 dense 子圖：`permute + view` 根本不在 trace 範圍內 + +`BEVFusionDenseWrapper.forward()` 的入口是 `lidar_bev`： + +```python +def forward(self, lidar_bev: torch.Tensor) -> tuple: + x = lidar_bev # 已經是 (N, C*D, H, W) 的 BEV feature map + x = self.mod.pts_backbone(x) + ... +``` + +**dense 子圖從一開始就接收已整理好的 BEV tensor，從來不會 trace 到 `ScatterND`、 +`dense()`、`permute`、`view` 這些操作。** 因此 dense ONNX 裡根本不存在這串節點， +不是「被刪掉」，而是「從未被畫進去」。 + +### 3.3 sparse 子圖：trace 時 batch=1 是靜態值，cleanup 再折疊 + +export 流程是先跑一次 `BEVFusionSparseWrapper` 的 forward pass 拿到 `lidar_bev`（`onnx_export_pipeline.py:307-313`）， +再用同樣那份 sample trace sparse ONNX。此時 batch=1 是 Python int， +`n = int(x.shape[0])` = `1`，trace 後 `Reshape` 的 shape 已是靜態常數 `[1, 256, 180, 180]`。 + +即使 trace 留下一些中介常數節點，`cleanup()` 做 constant folding 後也會把 +`Shape -> Gather -> Unsqueeze -> Concat` 這條鏈折成單一常數向量。 + +對照單圖 trace：整個圖一次畫完，batch 被宣告為 dynamic axis（或 symbolic 追蹤跨越較多節點邊界）， +`n` 無法靜態化，整串 shape plumbing 就保留下來。 + +### 3.4 小結 + +| | `Shape/Gather/Unsqueeze` 存在嗎？ | 原因 | +|---|---|---| +| 原始 single-trace ONNX | **有** | 整圖 trace，batch 為動態，`n` 要到 runtime 才讀 | +| split export 的 **dense 子圖** | **沒有**（完全不含這段程式） | dense 子圖從 `lidar_bev` 開始，`permute+view` 不在 trace 範圍 | +| split export 的 **sparse 子圖** + merge 後 | **通常沒有或更短** | trace 時 batch=1 為靜態整數；`cleanup()` constant folding | + +不是因為模型少算了，而是： +1. dense 子圖把 `lidar_bev` 當作外部輸入張量，`permute+view` 的 trace 根本不在這裡。 +2. sparse 子圖 trace 時 batch 維度靜態已知，shape plumbing 可被折疊。 +3. `cleanup()` 刪掉已無引用或可靜態化的 shape 節點。 + +**同一個數學流程，ONNX 可以有不同等價寫法；split trace + compose 通常更短。** + +--- + +## 4. 這會不會造成數值不同？ + +### 4.1 理論上 + +僅就 `Transpose/Shape/Gather/Unsqueeze/Reshape` 這類 layout/shape op： + +- 應是語義等價變換 +- 不涉及乘加，不應引入浮點誤差 + +### 4.2 實務上（真正會讓 mAP 掉到 0 的通常不是這串） + +真正高風險通常是契約不一致，例如： + +- `coors` 欄位順序契約（`xyz` vs `zyx`） +- BN fold 與 bias 契約（特別是 shadow export 路徑） +- head grid 尺寸契約（`grid_size // out_size_factor`，常見 180x180） + +也就是： +**節點長相不同通常不是主因；契約對不齊才是主因。** + +--- + +## 5. 如何判斷是不是「只是圖長相差異」 + +建議同一筆 sample 做分段比對： + +1. 比 `lidar_bev`（sparse 輸出）統計 +2. 比 `bbox_pred/score/label` 分布 +3. 若需要，再比最終 mAP + +如果中間張量與最終輸出一致，表示只是 ONNX 表達不同； +若差異巨大，優先檢查 `coors`/BN/head-grid 契約。 + +--- + +## 6. 結論 + +`ScatterND` 後面不一模一樣，主要是 **export 策略不同**： + +- 單圖 trace：保留較多動態 shape plumbing +- split trace + merge：圖更模組化，shape 子圖更容易被折疊 + +這種差異本身通常可接受；是否正確應以 **契約一致性** 與 **數值驗證** 為準，而不是只看節點數量或節點名稱是否一樣。 diff --git a/deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md b/deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md new file mode 100644 index 000000000..500ef5ecd --- /dev/null +++ b/deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md @@ -0,0 +1,127 @@ +# BEVFusion 2.8.x deployment notes + +What changed in BEVFusion **2.8.x** (model release commit `78b66a70`, *"feat(BEVFusion): +release BEVFusion 2.8.x (#217)"*) that matters for the new `deployment/` framework, and +how to run a 2.8 LiDAR model through the split FP16 export → TensorRT → eval pipeline. + +> The new `deployment/` framework was ported from the old (2.7-era) framework. This doc +> records the deltas needed to deploy a **2.8** checkpoint, validated end-to-end in Docker +> on 2026-06-30. + +--- + +## 1. How to run (split FP16, 2.8 model) + +```bash +# inside the awml-bevfusion container, /workspace = host AWML +python -m deployment.cli.main bevfusion \ + deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py \ + projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m_t4metric_v2.py +``` + +- Checkpoint: `work_dirs/bevfusion/bevfusion_2_8/best_epoch_25.pth` (7-class, set in the deploy config). +- Pair the checkpoint's class count with the model config. A **2.7** checkpoint + (`best_epoch_28.pth`) is **5-class** (`car, truck, bus, bicycle, pedestrian`); a **2.8** + checkpoint is **7-class** (adds `traffic_cone, barrier`). Mixing them gives + `size mismatch ... [5,...] vs [7,...]` on `bbox_head`. + +--- + +## 2. The 2.8 ONNX-export refactor (already in `projects/BEVFusion`) + +These changes live in the **model code** (commit `78b66a70`, already in the host repo). The +deployment wrappers call the model directly (`BEVFusionSparseWrapper.forward` → +`model.extract_pts_feat`; `BEVFusionDenseWrapper.forward` → `pts_backbone/neck` + +`bbox_head`), so the exported ONNX **automatically** reflects them. No deployment-side port +is required for these — they are listed so the graph shape is understood. + +| Area | 2.7 | 2.8 | File | +|---|---|---|---| +| Voxel mean-pool | `voxelize_reduce` inside `BEVFusion` (sum/`num_points`) | moved into a dedicated voxel encoder | `bevfusion.py` | +| Voxel feature encoding | sin-cos (`num_aug_features`) **inside** `BEVFusionSparseEncoder` | `HardSimpleVoxelSinCosEncoder` does mean-pool + sin-cos (folded `scale*x+bias` via `addcmul`) | `bevfusion_voxel_encoder.py` (new) | +| Sparse → dense | `out.dense()` + permute `(0,1,4,2,3)` | scatter-based `sparse_to_dense()` + permute `(0,4,3,1,2)` (cleaner ONNX, `dense_output_shapes`) | `custom_sparse_conv_tensor.py` (new), `sparse_encoder.py` | +| Head top-k | `argsort(...)[:num_proposals]` | `torch.topk(...)` (single TopK node) | `bevfusion_head.py` | +| Head local-max | in-place slice mutation `local_max[...] = ...` | `F.pad` + `torch.cat` + `local_concat_class_remapping` buffer (no in-place index assignment) | `bevfusion_head.py` | +| `bev_pos` | recomputed `.repeat().to(device)` | registered buffer, `query_pos = bev_pos.squeeze(0)[idx]` | `bevfusion_head.py` | + +Consequence: `BEVFusion.__init__` **no longer** has `voxelize_reduce` +(`voxelize_cfg.pop("voxelize_reduce")` and `assert self.voxelize_reduce` were removed). Do +**not** reintroduce them — a 2.7-era `bevfusion.py` will crash against a 2.8 config because +2.8 configs no longer set `voxelize_reduce`. + +`BEVFusionSparseEncoder.conv_out` output channels × `dense_output_shapes[2]` define the dense +`lidar_bev` channel count; for the j6gen2 base model this is **256**, with head BEV grid +**180×180** (`grid_size // out_size_factor = 1440 // 8`). These set the `bevfusion_dense` +TensorRT profile (`[1, 256, 180, 180]`). + +--- + +## 3. Deployment-side changes that WERE applied + +### 3.1 `max_num_points` 10 → 32 (TensorRT `voxels` profile) +2.8 voxelizes with `max_num_points = 32` (was 10). The `bevfusion_sparse` TensorRT profile's +`voxels` axis-1 must match, else the engine build fails with: + +``` +Dimension mismatch for tensor voxels and profile 0. +At dimension axis 1, profile has min=10, opt=10, max=10 but tensor has 32. +``` + +Fixed in `config/deploy_config_split_fp16_opt_2_8.py`: +```python +voxels=dict(min_shape=[1, 32, 5], opt_shape=[64000, 32, 5], max_shape=[256000, 32, 5]) +``` +(feature dim = **5** for the LiDAR+intensity model; the non-intensity model uses 4.) +This mirrors `projects/BEVFusion/deploy/utils.py` (`max_num_points 10 → 32`) and the +`*_tensorrt_dynamic.py` deploy configs in the 2.8 commit. + +### 3.2 ONNX opset 17 → 18 +2.8 bumped `opset_version` to **18** in +`projects/BEVFusion/configs/deploy/bevfusion_main_body_lidar_only*_tensorrt_dynamic.py`. +Applied to `config/deploy_config_split_fp16_opt_2_8.py` (`onnx_config.opset_version = 18`). + +### 3.3 What did NOT need porting +- **`purge_mmdeploy_symbolics(["layer_norm"])`** (added to `projects/BEVFusion/deploy/exporter.py` + in 2.8): only relevant to the **old** mmdeploy `RewriterContext` export path. The new + framework uses plain `torch.onnx.export` (`_torch_onnx_export_module`), so no mmdeploy + layer_norm symbolic is registered and `LayerNorm` exports natively. No-op here. +- **`_fix_topk`** (constant-K hardening so TensorRT gets a static `K`): still valid for the + 2.8 native `torch.topk` node — kept as is. + +--- + +## 4. The two `unknown` rows in the eval summary (`traffic_cone`, `barrier`) + +The eval summary prints `car, truck, bus, bicycle, pedestrian, unknown, unknown`. The two +`unknown` rows **are** `traffic_cone` and `barrier`. + +Cause (not a deployment bug): `perception_eval`'s `AutowareLabel` enum has only +`{UNKNOWN, CAR, TRUCK, BUS, BICYCLE, MOTORBIKE, PEDESTRIAN, ANIMAL, FP, LABEL_TYPE}`. Its +label table maps `movable_object.traffic_cone → UNKNOWN` and `movable_object.barrier → +UNKNOWN`. With `label_prefix="autoware"` (set by the model config's +`evaluation_config_dict`), both classes collapse to `UNKNOWN`, so they cannot be scored +separately. This is identical to what training-time `T4MetricV2` does in this container — it +is an evaluator/label-set limitation, not specific to deployment. The mAP for the 5 supported +classes is valid (e.g. car 0.95 / truck 0.97 / bus 0.99 / bicycle 0.99 / pedestrian 0.78). + +To score `traffic_cone`/`barrier` separately you would need a `perception_eval` label set +that includes them (or a non-autoware `label_prefix`); that is a model-repo / evaluator +change, out of scope for the deploy framework. + +--- + +## 5. Other framework-migration fixes (context) + +The `init migration` commit added `deployment/` but did not port the model-side deploy hooks +into `projects/`, and renamed some config APIs. To run 2.8 these were also needed: + +- `projects/SparseConvolution/sparse_functional.py` — `set_do_sort` / implicit-gemm + activation+bias fusion / `do_sort_i` export attr (the entrypoint imports `set_do_sort`). +- `projects/BEVFusion/bevfusion/bevfusion.py` — added `_align_lidar_bev_to_head_grid` + (the dense export wrapper calls it to assert BEV grid == head grid). **Only** this method + was added; the rest of the 2.8 `bevfusion.py` is unchanged. +- `deployment/projects/bevfusion/export/onnx_export_pipeline.py` — + `config.onnx_config` → `config.deploy_cfg.get("onnx_config", {})` (new framework keeps + `_onnx_config` private; only `get_onnx_settings(component)` is public). +- deploy config `runtime_io.info_file` → an existing `.pkl` + (`info/t4dataset_j6gen2_base_infos_test.pkl`). diff --git a/deployment/projects/bevfusion/docs/README.md b/deployment/projects/bevfusion/docs/README.md new file mode 100644 index 000000000..24eabe8b0 --- /dev/null +++ b/deployment/projects/bevfusion/docs/README.md @@ -0,0 +1,12 @@ +# BEVFusion deployment notes + +Deep-dive notes for the BEVFusion deployment pipeline (PyTorch → ONNX sparse+dense → TensorRT → +evaluation). The architecture map is in the parent [`README.md`](../README.md). + +| # | File | Topic | +|---|------|--------| +| 25 | [`25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md`](./25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md) | `coors` contract alignment with Autoware: why old/new ONNX both evaluate correctly, relation to ROS `x/y/z` and its boundaries | +| 26 | [`26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](./26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) | Why `ScatterND -> SECOND` differs between `original` and split-merge ONNX; why separate tracing needs less shape-plumbing; numerical impact | +| 28 | [`28_README_BEVFUSION_2_8_DEPLOYMENT.md`](./28_README_BEVFUSION_2_8_DEPLOYMENT.md) | BEVFusion 2.8.x deployment notes | + +Python entrypoints, configs, and pipelines live in the parent directory (`deployment/projects/bevfusion/`), alongside this `docs/` folder. diff --git a/deployment/projects/bevfusion/entrypoint.py b/deployment/projects/bevfusion/entrypoint.py new file mode 100644 index 000000000..ab6e130fa --- /dev/null +++ b/deployment/projects/bevfusion/entrypoint.py @@ -0,0 +1,174 @@ +"""BEVFusion deployment entrypoint invoked by the unified CLI.""" + +from __future__ import annotations + +import argparse +import logging + +from mmengine.config import Config + +from deployment.cli.args import add_deployment_file_logging, setup_logging +from deployment.config.base import BaseDeploymentConfig +from deployment.export.contexts import ExportContext +from deployment.projects.bevfusion.evaluation.evaluator import BEVFusionEvaluator +from deployment.projects.bevfusion.evaluation.executor import BEVFusionExecutor +from deployment.projects.bevfusion.io.component_utils import ( + has_component, + is_split_bevfusion_components, + maybe_add_merged_main_body_component, + should_merge_split_bevfusion, +) +from deployment.projects.bevfusion.io.data_loader import BEVFusionDataLoader +from deployment.projects.bevfusion.runner import BEVFusionDeploymentRunner +from deployment.projects.registry import project_registry + + +def _validate_bevfusion_components(config: BaseDeploymentConfig) -> None: + if is_split_bevfusion_components(config.components_cfg): + config.components_cfg.get_component("bevfusion_sparse") + config.components_cfg.get_component("bevfusion_dense") + if has_component(config.components_cfg, "bevfusion_main_body"): + config.components_cfg.get_component("bevfusion_main_body") + else: + config.components_cfg.get_component("bevfusion_main_body") + + +def _apply_bevfusion_component_merge_overlay( + config: BaseDeploymentConfig, + deploy_cfg: Config, + logger: logging.Logger, +) -> None: + """Apply optional split+merge overlay driven by deploy config.""" + if not should_merge_split_bevfusion(deploy_cfg): + return + before_names = list(config.components_cfg.component_names()) + config.components_cfg = maybe_add_merged_main_body_component( + deploy_cfg=deploy_cfg, + components_cfg=config.components_cfg, + ) + after_names = list(config.components_cfg.component_names()) + logger.info( + "BEVFusion merge flag enabled: keeping split export and adding merged artifacts component (%s -> %s)", + before_names, + after_names, + ) + + +def _extract_metrics_config(model_cfg: Config, logger: logging.Logger): + """Extract Detection3DMetricsConfig from model config. + + Tries T4MetricV2 first; falls back to a basic config if a different (or no) evaluator + is configured, so non-T4MetricV2 model configs still evaluate. + """ + from deployment.metrics.detection_3d_metrics import Detection3DMetricsConfig, extract_t4metric_v2_config + + class_names = model_cfg.class_names + + def _cfg_get(obj, key, default=None): + if obj is None: + return default + if isinstance(obj, dict): + return obj.get(key, default) + if key in obj: + return obj[key] + return getattr(obj, key, default) + + evaluator_cfg = getattr(model_cfg, "val_evaluator", None) or getattr(model_cfg, "test_evaluator", None) + if evaluator_cfg is None: + logger.warning("No evaluator config found; using basic metrics config") + return Detection3DMetricsConfig(class_names=class_names, frame_id="base_link") + + evaluator_type = getattr(evaluator_cfg, "type", None) + + if evaluator_type == "T4MetricV2": + return extract_t4metric_v2_config(model_cfg) + + perception_cfg = _cfg_get(evaluator_cfg, "perception_evaluator_configs") + frame_id = _cfg_get(evaluator_cfg, "frame_id") or _cfg_get(perception_cfg, "frame_id") or "base_link" + + logger.info( + "Evaluator type '%s'; using Detection3DMetricsConfig fallback (frame_id=%s)", + evaluator_type, + frame_id, + ) + return Detection3DMetricsConfig(class_names=class_names, frame_id=frame_id) + + +def _apply_spconv_do_sort(deploy_cfg: Config, logger: logging.Logger) -> None: + """Apply the ``spconv_do_sort`` field from ``deploy_cfg`` (default ``True``) to the + GetIndicePairsImplicitGemm symbolic/forward path. + + Controls the pair-mask argsort baked into the exported sparse graph; set ``False`` in a + deploy config to skip it. + """ + value = bool(deploy_cfg.get("spconv_do_sort", True)) + from projects.SparseConvolution.sparse_functional import set_do_sort + + set_do_sort(value) + logger.info( + "spconv_do_sort: %s (baked into GetIndicePairsImplicitGemm.do_sort_i at ONNX export)", + value, + ) + + +def run(args: argparse.Namespace) -> int: + """Run the BEVFusion deployment workflow.""" + deploy_cfg = Config.fromfile(args.deploy_cfg) + logger = setup_logging(args.log_level) + model_cfg = Config.fromfile(args.model_cfg) + config = BaseDeploymentConfig(deploy_cfg) + _apply_bevfusion_component_merge_overlay(config, deploy_cfg, logger) + + log_file = config.resolved_deploy_log_file + if log_file: + add_deployment_file_logging(log_file) + logger.info("Deployment log file: %s", log_file) + + project_registry.validate_required_components("bevfusion", config.components_cfg) + _validate_bevfusion_components(config) + _apply_spconv_do_sort(deploy_cfg, logger) + + logger.info("=" * 80) + logger.info("BEVFusion Deployment Pipeline") + logger.info("=" * 80) + + info_file = (deploy_cfg.get("runtime_io", {}) or {}).get("info_file", "") + data_loader = BEVFusionDataLoader( + info_file=info_file, + model_cfg=model_cfg, + ) + logger.info("Loaded %s samples", data_loader.num_samples) + + metrics_config = _extract_metrics_config(model_cfg, logger) + + plugin_libraries = tuple((deploy_cfg.get("tensorrt_config", {}) or {}).get("plugin_libraries", ()) or ()) + + # One executor instance, shared by the evaluator (evaluate/verify) and the runner + # (which hands it the loaded reference model after export). + executor = BEVFusionExecutor( + components_cfg=config.components_cfg, + tensorrt_plugin_libraries=plugin_libraries, + ) + + evaluator = BEVFusionEvaluator( + model_cfg=model_cfg, + metrics_config=metrics_config, + executor=executor, + ) + + module = getattr(args, "module", "main_body") + + runner = BEVFusionDeploymentRunner( + data_loader=data_loader, + evaluator=evaluator, + executor=executor, + config=config, + model_cfg=model_cfg, + deploy_cfg=deploy_cfg, + module=module, + plugin_libraries=plugin_libraries, + ) + + context = ExportContext() + runner.run(context=context) + return 0 diff --git a/deployment/projects/bevfusion/evaluation/__init__.py b/deployment/projects/bevfusion/evaluation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/projects/bevfusion/evaluation/evaluator.py b/deployment/projects/bevfusion/evaluation/evaluator.py new file mode 100644 index 000000000..94eaeb8f0 --- /dev/null +++ b/deployment/projects/bevfusion/evaluation/evaluator.py @@ -0,0 +1,107 @@ +"""BEVFusion evaluator for deployment. + +Thin subclass of ``Detection3DEvaluator``: only ``print_results`` (BEVFusion's indented +sparse/dense stage-wise latency layout) is BEVFusion-specific; the metrics hooks +(parse/accumulate/build/summarize) are shared with the base +(see ``deployment.evaluation.detection3d_evaluator``). +""" + +from __future__ import annotations + +import logging +from typing import Dict, Tuple + +from typing_extensions import override + +from deployment.evaluation.base_evaluator import EvalResultDict +from deployment.evaluation.detection3d_evaluator import Detection3DEvaluator + +logger = logging.getLogger(__name__) + +# (stage_key, indent_level): indent 0 = top-level; each +1 adds one leading space before the label. +_BEVFUSION_LATENCY_STAGE_LAYOUT: Tuple[Tuple[str, int], ...] = ( + ("preprocessing_ms", 0), + ("model_ms", 0), + ("bevfusion_ms", 0), + ("sparse_encoder_ms", 1), + ("dense_engine_ms", 1), + ("voxel_encoder_ms", 1), + ("backbone_ms", 2), + ("neck_ms", 2), + ("head_ms", 2), + ("post_scoring_ms", 2), + ("dense_unattributed_ms", 2), + ("postprocessing_ms", 0), +) + +_BEVFUSION_STAGE_DISPLAY_NAME: Dict[str, str] = { + "preprocessing_ms": "Preprocessing", + "model_ms": "Model", + "postprocessing_ms": "Postprocessing", + "bevfusion_ms": "Bevfusion", + "sparse_encoder_ms": "Sparse Encoder", + "dense_engine_ms": "Dense Engine", + "voxel_encoder_ms": "Voxel Encoder", + "backbone_ms": "Backbone", + "neck_ms": "Neck", + "head_ms": "Head", + "post_scoring_ms": "Post Scoring", + "dense_unattributed_ms": "Dense Unattributed", +} + + +def _bevfusion_stage_display_name(stage_key: str) -> str: + return _BEVFUSION_STAGE_DISPLAY_NAME.get( + stage_key, + stage_key.replace("_ms", "").replace("_", " ").title(), + ) + + +class BEVFusionEvaluator(Detection3DEvaluator): + """Evaluator for BEVFusion 3D detection deployment.""" + + @override + def print_results(self, results: EvalResultDict) -> None: + """Log the metrics report, latency statistics, and BEVFusion's indented breakdown.""" + metrics_report = self.metrics_interface.format_metrics_report() + if metrics_report: + for line in metrics_report.rstrip().split("\n"): + logger.info(line) + + if "latency" in results: + self._log_latency_stats(results) + + if "latency_breakdown" in results: + breakdown = results["latency_breakdown"] + breakdown_dict = breakdown.to_dict() if hasattr(breakdown, "to_dict") else breakdown + if breakdown_dict: + logger.info("") + logger.info("Stage-wise Latency Breakdown:") + printed: set[str] = set() + for stage_key, indent_level in _BEVFUSION_LATENCY_STAGE_LAYOUT: + if stage_key not in breakdown_dict: + continue + stats = breakdown_dict[stage_key] + stats_dict = stats.to_dict() if hasattr(stats, "to_dict") else stats + mean_ms = stats_dict.get("mean_ms", 0.0) + std_ms = stats_dict.get("std_ms", 0.0) + if mean_ms == 0.0 and std_ms == 0.0: + continue + printed.add(stage_key) + prefix = " " * (2 + indent_level) + label = _bevfusion_stage_display_name(stage_key) + logger.info("%s%-18s: %.2f ± %.2f ms", prefix, label, mean_ms, std_ms) + + extra_keys = sorted(k for k in breakdown_dict if k not in printed) + for stage_key in extra_keys: + stats = breakdown_dict[stage_key] + stats_dict = stats.to_dict() if hasattr(stats, "to_dict") else stats + mean_ms = stats_dict.get("mean_ms", 0.0) + std_ms = stats_dict.get("std_ms", 0.0) + if mean_ms == 0.0 and std_ms == 0.0: + continue + label = _bevfusion_stage_display_name(stage_key) + logger.info(" %-18s: %.2f ± %.2f ms", label, mean_ms, std_ms) + + logger.info("") + logger.info("Total Samples: %s", results["num_samples"]) diff --git a/deployment/projects/bevfusion/evaluation/executor.py b/deployment/projects/bevfusion/evaluation/executor.py new file mode 100644 index 000000000..c2c1974e7 --- /dev/null +++ b/deployment/projects/bevfusion/evaluation/executor.py @@ -0,0 +1,58 @@ +"""BEVFusion backend executor. + +Thin subclass of ``PointDetectionExecutor``: declares the BEVFusion pipeline classes, the +split/merged output-name lookup, and forwards custom spconv ImplicitGemm ``plugin_libraries`` to the +TensorRT pipeline. Pipeline creation and ``(points, metainfo)`` input prep are shared with the +base (see ``deployment.evaluation.point_detection_executor``). + +This replaces the OLD ``BEVFusionPipelineFactory`` (the global pipeline registry was removed in +the refactor): pipeline construction uses the reference model on ``self.pytorch_model`` (set by +the runner after export). +""" + +from typing import Any, Iterable, List, Mapping, Optional + +from typing_extensions import override + +from deployment.config.schema import ComponentsConfig +from deployment.evaluation.point_detection_executor import PointDetectionExecutor +from deployment.projects.bevfusion.inference.onnx_inference_pipeline import BEVFusionONNXPipeline +from deployment.projects.bevfusion.inference.pytorch_inference_pipeline import BEVFusionPyTorchPipeline +from deployment.projects.bevfusion.inference.tensorrt_inference_pipeline import BEVFusionTensorRTPipeline +from deployment.projects.bevfusion.io.component_utils import has_component, is_split_bevfusion_components + + +class BEVFusionExecutor(PointDetectionExecutor): + """Backend execution primitives for BEVFusion (pipeline creation, input prep). + + Args: + components_cfg: Unified components configuration, forwarded to the ONNX/TensorRT + pipelines so they can resolve split (sparse+dense) vs merged main-body artifacts. + tensorrt_plugin_libraries: Custom TensorRT plugin ``.so`` paths forwarded to the + TensorRT pipeline (e.g. the spconv ImplicitGemm plugin); empty when none is needed. + """ + + task_name = "BEVFusion" + pytorch_pipeline_cls = BEVFusionPyTorchPipeline + onnx_pipeline_cls = BEVFusionONNXPipeline + tensorrt_pipeline_cls = BEVFusionTensorRTPipeline + + def __init__(self, components_cfg: ComponentsConfig, tensorrt_plugin_libraries: Iterable[str] = ()) -> None: + super().__init__(components_cfg) + self._tensorrt_plugin_libraries = tuple(tensorrt_plugin_libraries) + + @override + def _tensorrt_pipeline_kwargs(self) -> Mapping[str, Any]: + """Forward the custom spconv ImplicitGemm plugin ``.so`` paths to the TensorRT pipeline.""" + return {"plugin_libraries": self._tensorrt_plugin_libraries} + + @override + def get_output_names(self) -> Optional[List[str]]: + """Return the model output names (split→dense outputs; otherwise main-body outputs).""" + if is_split_bevfusion_components(self._components_cfg) and not has_component( + self._components_cfg, "bevfusion_main_body" + ): + comp = self._components_cfg.get_component("bevfusion_dense") + else: + comp = self._components_cfg.get_component("bevfusion_main_body") + return [out.name for out in comp.io.outputs] diff --git a/deployment/projects/bevfusion/export/__init__.py b/deployment/projects/bevfusion/export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/projects/bevfusion/export/onnx_export_pipeline.py b/deployment/projects/bevfusion/export/onnx_export_pipeline.py new file mode 100644 index 000000000..08ba2495b --- /dev/null +++ b/deployment/projects/bevfusion/export/onnx_export_pipeline.py @@ -0,0 +1,732 @@ +"""BEVFusion ONNX export pipeline. + +Exports the BEVFusion main_body to a single ONNX file, including the TopK fix. +Replicates the logic from projects/BEVFusion/deploy/ within the new deployment framework. +""" + +from __future__ import annotations + +import contextlib +import logging +import os +import warnings +from pathlib import Path +from typing import Any, Dict, Optional + +import numpy as np +import onnx +import onnx_graphsurgeon as gs +import torch +import torch.nn as nn +import torch.nn.functional as F + +from deployment.config.base import BaseDeploymentConfig +from deployment.io.base_data_loader import BaseDataLoader +from deployment.primitives.artifacts import Artifact +from deployment.projects.bevfusion.io.component_utils import ( + has_component, + is_split_bevfusion_components, + should_merge_split_bevfusion, +) + +logger = logging.getLogger(__name__) + + +def _normalize_sparse_coors_for_autoware(coors: torch.Tensor) -> torch.Tensor: + """Normalize sparse coordinates to the legacy Autoware export contract. + + Graph **inputs** must be ``[z, y, x]`` (no batch). This wrapper flips to + ``[x, y, z]`` and prepends batch — same as ``projects/BEVFusion/deploy/containers.py``. + Voxelization outputs ``[x, y, z]``; convert with ``voxel_indices_xyz_to_graph_input_zyx`` + before tracing or feeding ONNX/TRT. + """ + from deployment.projects.bevfusion.io.coors_contract import graph_input_zyx_to_model_indices_xyz + + coors = coors.to(dtype=torch.int32) + if coors.shape[1] == 3: + num_points = coors.shape[0] + coors = graph_input_zyx_to_model_indices_xyz(coors) + batch_coors = torch.zeros(num_points, 1, dtype=torch.int32, device=coors.device) + coors = torch.cat([batch_coors, coors], dim=1).contiguous() + return coors + + +def _head_dict_to_export_outputs(outputs: dict) -> tuple: + """Turn the detection-head output dict into the (bbox_pred, score, label) ONNX outputs.""" + score = outputs["heatmap"].sigmoid() + one_hot = F.one_hot(outputs["query_labels"], num_classes=score.size(1)).permute(0, 2, 1) + score = score * outputs["query_heatmap_score"] * one_hot + score = score[0].max(dim=0)[0] + + bbox_pred = torch.cat( + [outputs["center"][0], outputs["height"][0], outputs["dim"][0], outputs["rot"][0], outputs["vel"][0]], + dim=0, + ) + + return bbox_pred, score, outputs["query_labels"][0] + + +class BEVFusionSparseWrapper(nn.Module): + """LiDAR sparse tower only: voxels/coors/num_points → BEV feature map.""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.mod = model + + def forward( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> torch.Tensor: + voxels = voxels.to(dtype=torch.float32) + coors = _normalize_sparse_coors_for_autoware(coors) + + return self.mod.extract_pts_feat(voxels, coors, num_points_per_voxel, points=None) + + +class BEVFusionDenseWrapper(nn.Module): + """SECOND + neck + head (+ ONNX postprocess). Input: ``lidar_bev`` [B,C,H,W].""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.mod = model + + def forward(self, lidar_bev: torch.Tensor) -> tuple: + x = lidar_bev + if self.mod.pts_backbone is not None: + x = self.mod.pts_backbone(x) + if self.mod.pts_neck is not None: + x = self.mod.pts_neck(x) + x = self.mod._align_lidar_bev_to_head_grid(x) + outputs = self.mod.bbox_head(x, []) + head_out = outputs[0][0] + return _head_dict_to_export_outputs(head_out) + + +class BEVFusionMainBodyWrapper(nn.Module): + """Wrapper for BEVFusion that matches the ONNX export interface. + + Takes voxels/coors/num_points_per_voxel and returns bbox_pred/score/label_pred. + Replicates TrtBevFusionMainContainer from projects/BEVFusion/deploy/containers.py. + """ + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.mod = model + + def forward( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> tuple: + # spconv requires int32 indices; float batch column (torch.zeros default) + int coors + # yields float tensor and can CUDA fault in implicit_gemm. Keep voxels FP32. + voxels = voxels.to(dtype=torch.float32) + coors = _normalize_sparse_coors_for_autoware(coors) + + batch_inputs_dict = { + "voxels": {"voxels": voxels, "coors": coors, "num_points_per_voxel": num_points_per_voxel}, + } + + outputs = self.mod._forward(batch_inputs_dict, using_image_features=True) + return _head_dict_to_export_outputs(outputs) + + +class BEVFusionONNXExportPipeline: + """ONNX export for BEVFusion. + + - **Single-file** (``bevfusion_main_body``): full graph, TopK fix applied. + - **Split** (``bevfusion_sparse`` + ``bevfusion_dense``): sparse tower ONNX + dense ONNX + (route 1: sparse can go to libspconv / plugin; dense → TensorRT without spconv ops). + """ + + def __init__( + self, + module: str = "main_body", + logger: Optional[logging.Logger] = None, + ) -> None: + self.module = module + self.logger = logger or logging.getLogger(__name__) + + def export( + self, + *, + model: torch.nn.Module, + data_loader: BaseDataLoader, + output_dir: str, + config: BaseDeploymentConfig, + sample_idx: int = 0, + ) -> Artifact: + output_dir_path = Path(output_dir) + output_dir_path.mkdir(parents=True, exist_ok=True) + + if is_split_bevfusion_components(config.components_cfg): + return self._export_split(model, data_loader, output_dir_path, config, sample_idx) + + self.logger.info("=" * 80) + self.logger.info("Exporting BEVFusion to ONNX (single-file)") + self.logger.info("=" * 80) + + device = next(model.parameters()).device + self.logger.info(f"Model device: {device}") + + component_cfg = config.components_cfg.get_component("bevfusion_main_body") + onnx_filename = component_cfg.onnx_file + output_path = output_dir_path / onnx_filename + temp_path = output_dir_path / onnx_filename.replace(".onnx", "_temp_to_be_fixed.onnx") + + self.logger.info(f"Loading sample {sample_idx} for export tracing...") + sample = data_loader.load_sample(sample_idx) + points = sample["points"] + self.logger.info(f"Sample loaded: points shape={points.shape}") + + self.logger.info("Running voxelization...") + voxels, coors, num_points_per_voxel = self._voxelize(model, points) + self.logger.info(f"Voxelization done: {voxels.shape[0]} voxels") + + onnx_cfg = self._get_onnx_config(config, "bevfusion_main_body") + self.logger.info( + f"ONNX config: opset={onnx_cfg['opset_version']}, inputs={onnx_cfg['input_names']}, outputs={onnx_cfg['output_names']}" + ) + + self.logger.info("Running torch.onnx.export...") + self._export_to_onnx( + model, + voxels, + coors, + num_points_per_voxel, + str(temp_path), + onnx_cfg, + fuse_spconv_bn=bool(config.deploy_cfg.get("fuse_spconv_bn", False)), + ) + + num_proposals = self._get_num_proposals(model) + self._fix_topk(str(temp_path), str(output_path), num_proposals) + + self.logger.info("=" * 80) + self.logger.info(f"BEVFusion ONNX export successful: {output_path}") + self.logger.info("=" * 80) + + return Artifact(path=str(output_dir_path)) + + def _export_split( + self, + model: torch.nn.Module, + data_loader: BaseDataLoader, + output_dir_path: Path, + config: BaseDeploymentConfig, + sample_idx: int, + ) -> Artifact: + """Export ``bevfusion_sparse.onnx`` and ``bevfusion_dense.onnx``.""" + self.logger.info("=" * 80) + self.logger.info("Exporting BEVFusion to ONNX (split: sparse + dense)") + self.logger.info("=" * 80) + + self._assert_split_model_ok(model) + + device = next(model.parameters()).device + self.logger.info(f"Model device: {device}") + + self.logger.info(f"Loading sample {sample_idx} for export tracing...") + sample = data_loader.load_sample(sample_idx) + points = sample["points"] + self.logger.info(f"Sample loaded: points shape={points.shape}") + + self.logger.info("Running voxelization...") + voxels, coors, num_points_per_voxel = self._voxelize(model, points) + self.logger.info(f"Voxelization done: {voxels.shape[0]} voxels") + + sparse_cfg = config.components_cfg.get_component("bevfusion_sparse") + dense_cfg = config.components_cfg.get_component("bevfusion_dense") + sparse_onnx = output_dir_path / sparse_cfg.onnx_file + dense_onnx = output_dir_path / dense_cfg.onnx_file + dense_temp = output_dir_path / dense_cfg.onnx_file.replace(".onnx", "_temp_to_be_fixed.onnx") + + onnx_cfg_sparse = self._get_onnx_config(config, "bevfusion_sparse") + onnx_cfg_dense = self._get_onnx_config(config, "bevfusion_dense") + + self.logger.info( + "Sparse ONNX: inputs=%s outputs=%s", + onnx_cfg_sparse["input_names"], + onnx_cfg_sparse["output_names"], + ) + self.logger.info("Running torch.onnx.export (sparse)...") + self._export_to_onnx( + model, + voxels, + coors, + num_points_per_voxel, + str(sparse_onnx), + onnx_cfg_sparse, + wrapper="sparse", + fuse_spconv_bn=bool(config.deploy_cfg.get("fuse_spconv_bn", False)), + ) + self._postprocess_sparse_onnx_fp(config=config, sparse_onnx_path=sparse_onnx) + + with torch.no_grad(): + sw = BEVFusionSparseWrapper(model) + sw.eval() + trace_dev = device + lidar_bev = sw( + voxels.to(trace_dev), coors.to(trace_dev, dtype=torch.int32), num_points_per_voxel.to(trace_dev) + ) + self.logger.info("Dense trace input lidar_bev shape: %s", tuple(lidar_bev.shape)) + + self.logger.info( + "Dense ONNX: inputs=%s outputs=%s", + onnx_cfg_dense["input_names"], + onnx_cfg_dense["output_names"], + ) + self.logger.info("Running torch.onnx.export (dense)...") + self._export_dense_to_onnx(model, lidar_bev, str(dense_temp), onnx_cfg_dense) + + num_proposals = self._get_num_proposals(model) + self._fix_topk(str(dense_temp), str(dense_onnx), num_proposals) + + if should_merge_split_bevfusion(config.deploy_cfg): + self._merge_split_onnx_artifact( + config=config, + sparse_onnx_path=sparse_onnx, + dense_onnx_path=dense_onnx, + output_dir_path=output_dir_path, + ) + + self.logger.info("=" * 80) + self.logger.info("Split ONNX export OK: %s , %s", sparse_onnx, dense_onnx) + self.logger.info("=" * 80) + + return Artifact(path=str(output_dir_path)) + + @staticmethod + def _deploy_cfg_fuse_implicit_gemm_relu(deploy_cfg: Any, *, default: bool = True) -> bool: + """Read ``spconv_fuse_implicit_gemm_relu`` (fuse trailing Relu into ImplicitGemm nodes).""" + val = deploy_cfg.get("spconv_fuse_implicit_gemm_relu", None) + if val is not None: + return bool(val) + return default + + def _postprocess_sparse_onnx_fp(self, *, config: BaseDeploymentConfig, sparse_onnx_path: Path) -> None: + """Optional FP sparse ONNX postprocess (ImplicitGemm activation fusion).""" + enable_fuse = self._deploy_cfg_fuse_implicit_gemm_relu(config.deploy_cfg, default=False) + if not enable_fuse: + self.logger.info("Sparse ONNX postprocess: ImplicitGemm ReLU fuse disabled by deploy config.") + return + if not sparse_onnx_path.exists(): + raise FileNotFoundError(f"Sparse ONNX not found for postprocess: {sparse_onnx_path}") + + from deployment.projects.bevfusion.export.onnx_fuse_implicit_gemm_activation import ( + fuse_autoware_implicit_gemm_trailing_relu, + ) + + model = onnx.load(str(sparse_onnx_path)) + n_relu = fuse_autoware_implicit_gemm_trailing_relu(model) + onnx.save_model(model, str(sparse_onnx_path)) + + self.logger.info( + "Sparse ONNX postprocess: ImplicitGemm fuse done (trailing Relu=%d): %s", + n_relu, + sparse_onnx_path, + ) + + def _merge_split_onnx_artifact( + self, + *, + config: BaseDeploymentConfig, + sparse_onnx_path: Path, + dense_onnx_path: Path, + output_dir_path: Path, + ) -> None: + """Merge split sparse+dense ONNX into single main_body ONNX.""" + if not has_component(config.components_cfg, "bevfusion_main_body"): + raise KeyError( + "bevfusion_merge is enabled but components_cfg has no 'bevfusion_main_body'. " + "Ensure merge overlay is applied before export." + ) + merged_cfg = config.components_cfg.get_component("bevfusion_main_body") + merged_path = output_dir_path / merged_cfg.onnx_file + + try: + from onnx import compose as onnx_compose + except Exception as e: + raise RuntimeError("ONNX compose utilities unavailable; cannot merge split ONNX.") from e + + if not sparse_onnx_path.exists(): + raise FileNotFoundError(f"Sparse ONNX not found: {sparse_onnx_path}") + if not dense_onnx_path.exists(): + raise FileNotFoundError(f"Dense ONNX not found: {dense_onnx_path}") + + sparse_model = onnx.load(str(sparse_onnx_path)) + dense_model = onnx.load(str(dense_onnx_path)) + + # onnx.compose.merge_models requires identical IR/opset metadata. + target_ir = max(int(sparse_model.ir_version), int(dense_model.ir_version)) + sparse_model.ir_version = target_ir + dense_model.ir_version = target_ir + + sparse_opsets = {imp.domain: int(imp.version) for imp in sparse_model.opset_import} + dense_opsets = {imp.domain: int(imp.version) for imp in dense_model.opset_import} + merged_opsets = dict(sparse_opsets) + for domain, version in dense_opsets.items(): + merged_opsets[domain] = max(version, merged_opsets.get(domain, version)) + merged_opset_ids = [onnx.helper.make_operatorsetid(d, v) for d, v in merged_opsets.items()] + del sparse_model.opset_import[:] + sparse_model.opset_import.extend(merged_opset_ids) + del dense_model.opset_import[:] + dense_model.opset_import.extend(merged_opset_ids) + + sparse_pref = onnx_compose.add_prefix(sparse_model, prefix="sparse/") + dense_pref = onnx_compose.add_prefix(dense_model, prefix="dense/") + + sparse_out_name = config.components_cfg.get_component("bevfusion_sparse").io.outputs[0].name + dense_in_name = config.components_cfg.get_component("bevfusion_dense").io.inputs[0].name + io_map = [(f"sparse/{sparse_out_name}", f"dense/{dense_in_name}")] + + merged_model = onnx_compose.merge_models(sparse_pref, dense_pref, io_map=io_map) + merged_graph = gs.import_onnx(merged_model) + + sparse_inputs = [inp.name for inp in config.components_cfg.get_component("bevfusion_sparse").io.inputs] + dense_outputs = [out.name for out in config.components_cfg.get_component("bevfusion_dense").io.outputs] + if len(merged_graph.inputs) != len(sparse_inputs): + self.logger.warning( + "Merged ONNX input count mismatch: graph=%d expected=%d", + len(merged_graph.inputs), + len(sparse_inputs), + ) + if len(merged_graph.outputs) != len(dense_outputs): + self.logger.warning( + "Merged ONNX output count mismatch: graph=%d expected=%d", + len(merged_graph.outputs), + len(dense_outputs), + ) + for i, name in enumerate(sparse_inputs): + if i < len(merged_graph.inputs): + merged_graph.inputs[i].name = name + for i, name in enumerate(dense_outputs): + if i < len(merged_graph.outputs): + merged_graph.outputs[i].name = name + + merged_graph.cleanup().toposort() + onnx.save_model(gs.export_onnx(merged_graph), str(merged_path)) + self.logger.info("Merged split ONNX -> %s", merged_path) + + @staticmethod + def _assert_split_model_ok(model: torch.nn.Module) -> None: + if getattr(model, "fusion_layer", None) is not None: + raise RuntimeError( + "Split ONNX export requires LiDAR-only path (fusion_layer must be None). " + "Use single-file export or implement a fusion ONNX branch." + ) + if getattr(model, "img_backbone", None) is not None: + raise RuntimeError("Split ONNX export is for LiDAR-only BEVFusion (img_backbone must be None).") + if getattr(model, "pts_middle_encoder", None) is None: + raise RuntimeError("pts_middle_encoder is required for split sparse export.") + + def _voxelize(self, model: torch.nn.Module, points: torch.Tensor) -> tuple: + """Run voxelization on a point cloud sample.""" + device = next(model.parameters()).device + points = points.to(device).float() + + with torch.no_grad(): + ret = model.pts_voxel_layer(points) + if len(ret) == 3: + feats, coords, sizes = ret + else: + feats, coords = ret + sizes = torch.ones(feats.shape[0], device=device) + from deployment.projects.bevfusion.io.coors_contract import voxel_indices_xyz_to_graph_input_zyx + + coords = coords[:, :].to(dtype=torch.int32) # [M, 3] (x, y, z) from voxel layer + coords = voxel_indices_xyz_to_graph_input_zyx(coords) # ONNX graph input: [z, y, x] + + return feats, coords, sizes + + def _get_onnx_config(self, config: BaseDeploymentConfig, component_name: str) -> Dict[str, Any]: + """Build ONNX export configuration for a components_cfg entry.""" + component_cfg = config.components_cfg.get_component(component_name) + io_cfg = component_cfg.io + + input_names = [inp.name for inp in io_cfg.inputs] + output_names = [out.name for out in io_cfg.outputs] + + dynamic_axes = {} + if hasattr(io_cfg, "dynamic_axes") and io_cfg.dynamic_axes: + dynamic_axes = dict(io_cfg.dynamic_axes) + + onnx_settings = config.deploy_cfg.get("onnx_config", {}) or {} + opset_version = getattr(onnx_settings, "opset_version", 17) + # Default "auto" = trace on the same device as the model (usually CUDA). CUDA-built spconv + # implicit_gemm runs GPU kernels: indices/features on CPU + those kernels => cudaErrorIllegalAddress. + # If you lack GPU memory for dense(), set trace_device=cpu only with a CPU spconv build, or export + # on another machine. + trace_device = getattr(onnx_settings, "trace_device", None) or os.environ.get( + "BEVFUSION_ONNX_TRACE_DEVICE", "auto" + ) + + return { + "input_names": input_names, + "output_names": output_names, + "dynamic_axes": dynamic_axes, + "opset_version": opset_version, + "do_constant_folding": bool(getattr(onnx_settings, "do_constant_folding", True)), + "export_params": True, + "keep_initializers_as_inputs": False, + "verbose": False, + "trace_device": trace_device, + } + + def _torch_onnx_export_module( + self, + module: nn.Module, + model_inputs: tuple, + output_path: str, + onnx_cfg: Dict[str, Any], + ) -> None: + """Run ``torch.onnx.export`` with deploy ``onnx_config`` (incl. ``do_constant_folding``).""" + export_kw: Dict[str, Any] = dict( + export_params=onnx_cfg["export_params"], + input_names=onnx_cfg["input_names"], + output_names=onnx_cfg["output_names"], + opset_version=onnx_cfg["opset_version"], + dynamic_axes=onnx_cfg["dynamic_axes"], + keep_initializers_as_inputs=onnx_cfg["keep_initializers_as_inputs"], + verbose=onnx_cfg["verbose"], + do_constant_folding=bool(onnx_cfg.get("do_constant_folding", True)), + ) + try: + from torch.onnx import TrainingMode + + export_kw["training"] = TrainingMode.EVAL + except Exception: + pass + + with torch.no_grad(): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=".*non-tuple sequence for multidimensional indexing.*", + category=UserWarning, + ) + torch.onnx.export(module, model_inputs, output_path, **export_kw) + + def _resolve_trace_device( + self, + model_device: torch.device, + onnx_cfg: Dict[str, Any], + *, + warn_on_cpu: bool = False, + ) -> torch.device: + """Resolve the tracing device from ``onnx_cfg``, coercing CPU back to the model's GPU. + + CUDA-built spconv implicit_gemm cannot run with CPU indices (merge_sort illegal + address), so a requested ``cpu`` trace device is overridden to the model's CUDA device. + """ + raw_td = onnx_cfg.get("trace_device") or "auto" + trace_dev = model_device if raw_td in ("auto", "", None) else torch.device(raw_td) + + if model_device.type == "cuda" and trace_dev.type == "cpu": + if warn_on_cpu: + self.logger.warning( + "trace_device=cpu while model is on %s: CUDA spconv implicit_gemm does not support " + "CPU indices (merge_sort illegal address). Tracing on %s instead. " + "For large dense() OOM, use a larger GPU or export elsewhere; do not use CPU trace with CUDA spconv.", + model_device, + model_device, + ) + trace_dev = model_device + return trace_dev + + @contextlib.contextmanager + def _model_on_trace_device( + self, + model: torch.nn.Module, + model_device: torch.device, + trace_dev: torch.device, + ): + """Temporarily move ``model`` to ``trace_dev`` for tracing, restoring it afterward.""" + moved = trace_dev != model_device + if moved: + self.logger.info( + "Moving model to %s for ONNX tracing (model was on %s; avoids GPU OOM from sparse dense()).", + trace_dev, + model_device, + ) + model.to(trace_dev) + try: + yield + finally: + if moved: + try: + model.to(model_device) + except Exception as e: + self.logger.warning( + "Could not move model back to %s after ONNX export (GPU may be in error state): %s", + model_device, + e, + ) + if model_device.type == "cuda": + try: + torch.cuda.empty_cache() + except Exception: + pass + + def _export_dense_to_onnx( + self, + model: torch.nn.Module, + lidar_bev: torch.Tensor, + output_path: str, + onnx_cfg: Dict[str, Any], + ) -> None: + """Export pts_backbone + neck + head (+ postprocess) to ONNX.""" + model_device = next(model.parameters()).device + trace_dev = self._resolve_trace_device(model_device, onnx_cfg) + + with self._model_on_trace_device(model, model_device, trace_dev): + wrapper = BEVFusionDenseWrapper(model) + wrapper.eval() + wrapper.to(trace_dev) + bev = lidar_bev.to(trace_dev) + self._torch_onnx_export_module(wrapper, (bev,), output_path, onnx_cfg) + + self.logger.info("Exported dense ONNX to %s", output_path) + + def _export_to_onnx( + self, + model: torch.nn.Module, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + output_path: str, + onnx_cfg: Dict[str, Any], + *, + wrapper: str = "main", + fuse_spconv_bn: bool = False, + ) -> None: + """Export voxel-based subgraph to ONNX (full main_body or sparse-only).""" + model_device = next(model.parameters()).device + trace_dev = self._resolve_trace_device(model_device, onnx_cfg, warn_on_cpu=True) + + with self._model_on_trace_device(model, model_device, trace_dev): + orig_sparse_encoder: Optional[nn.Module] = None + try: + orig_sparse_encoder = self._maybe_swap_in_float_shadow_encoder( + model, trace_dev, fuse_spconv_bn=fuse_spconv_bn + ) + + if wrapper == "sparse": + wrapper_mod: nn.Module = BEVFusionSparseWrapper(model) + elif wrapper == "main": + wrapper_mod = BEVFusionMainBodyWrapper(model) + else: + raise ValueError(f"Unknown wrapper '{wrapper}' for ONNX export") + + model_inputs = ( + voxels.to(trace_dev), + coors.to(device=trace_dev, dtype=torch.int32), + num_points_per_voxel.to(trace_dev), + ) + wrapper_mod.eval() + wrapper_mod.to(trace_dev) + + self._torch_onnx_export_module(wrapper_mod, model_inputs, output_path, onnx_cfg) + finally: + if orig_sparse_encoder is not None: + model.pts_middle_encoder = orig_sparse_encoder + + self.logger.info("Exported ONNX to %s", output_path) + + def _maybe_swap_in_float_shadow_encoder( + self, + model: torch.nn.Module, + trace_dev: torch.device, + *, + fuse_spconv_bn: bool, + ) -> Optional[nn.Module]: + """Swap ``pts_middle_encoder`` for a fused FP32 shadow used only during tracing. + + Returns the original encoder (to restore after export) or ``None`` if no swap was + needed. The shadow lets BN be folded (``fuse_spconv_bn``) into a clean sparse ONNX + graph without mutating the runtime model. + """ + enc = getattr(model, "pts_middle_encoder", None) + if enc is None: + return None + + from deployment.projects.bevfusion.export.sparse_encoder_float_shadow import ( + build_float_sparse_encoder_shadow, + resolve_sparse_onnx_shadow, + ) + + gm_src, cfg_ov = resolve_sparse_onnx_shadow(enc, model) + if gm_src is None: + return None + + self.logger.info( + "Sparse tower: using fused FP32 shadow encoder for ONNX export " + "(weights copied from source sparse encoder)." + ) + if cfg_ov: + self.logger.info( + "Shadow rebuild merges %d attribute(s) from model.cfg pts_middle_encoder.", + len(cfg_ov), + ) + + model.pts_middle_encoder = build_float_sparse_encoder_shadow( + gm_src, + trace_dev, + cfg_overrides=cfg_ov if cfg_ov else None, + fuse_spconv_bn=bool(fuse_spconv_bn), + ) + return enc + + def _get_num_proposals(self, model: torch.nn.Module) -> int: + """Extract num_proposals from the BEVFusion model config.""" + cfg = getattr(model, "cfg", None) + if cfg is not None: + num_proposals = cfg.get("num_proposals", None) + if num_proposals is not None: + return int(num_proposals) + + if hasattr(model, "bbox_head") and hasattr(model.bbox_head, "num_proposals"): + return int(model.bbox_head.num_proposals) + + raise ValueError( + "num_proposals not found in model config or bbox_head. " + "Ensure model_cfg or bbox_head.num_proposals is set." + ) + + def _fix_topk(self, input_path: str, output_path: str, num_proposals: int) -> None: + """Fix the TopK node in the ONNX graph to use a constant K. + + TensorRT requires TopK's K to be a constant, but torch.onnx.export + may produce a dynamic K. This replaces it with num_proposals. + """ + self.logger.info(f"Fixing TopK (K={num_proposals}) in ONNX graph...") + model = onnx.load(input_path) + graph = gs.import_onnx(model) + + topk_nodes = [node for node in graph.nodes if node.op == "TopK"] + if len(topk_nodes) == 0: + self.logger.warning("No TopK node found; skipping fix") + onnx.save_model(model, output_path) + return + + if len(topk_nodes) != 1: + self.logger.warning(f"Expected 1 TopK node, found {len(topk_nodes)}; fixing the first one") + + topk = topk_nodes[0] + topk.inputs[1] = gs.Constant("K", values=np.array([num_proposals], dtype=np.int64)) + topk.outputs[0].shape = [1, num_proposals] + topk.outputs[0].dtype = topk.inputs[0].dtype if topk.inputs[0].dtype else np.float32 + topk.outputs[1].shape = [1, num_proposals] + topk.outputs[1].dtype = np.int64 + + graph.cleanup().toposort() + onnx.save_model(gs.export_onnx(graph), output_path) + + # Clean up temp file + if os.path.exists(input_path) and input_path != output_path: + os.remove(input_path) + + self.logger.info(f"TopK fixed. Final ONNX: {output_path}") diff --git a/deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py b/deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py new file mode 100644 index 000000000..207db2088 --- /dev/null +++ b/deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py @@ -0,0 +1,148 @@ +"""Fuse post-spconv activation into ``autoware`` ImplicitGemm plugin ``act_type``. + +TensorRT does not fuse standard ONNX ``Relu`` with custom ops, so we fold +``ImplicitGemm → Relu`` by setting ``act_type=kReLU`` on the producer node and +removing the standalone ``Relu`` node. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import DefaultDict, Dict, List, Optional, Set + +import numpy as np +import onnx +from onnx import helper, numpy_helper + + +def _normalize_attr(name: str) -> str: + """Strip ONNX type suffix (``_f``, ``_i``, ``_s``, ``_l``) from an attribute name.""" + for suf in ("_f", "_i", "_s", "_l"): + if name.endswith(suf) and len(name) > len(suf): + return name[: -len(suf)] + return name + + +def _try_get_constant_numpy( + graph: onnx.GraphProto, + name: str, + init_map: Dict[str, np.ndarray], +) -> Optional[np.ndarray]: + """Return the constant numpy array for tensor ``name``, or ``None`` if not a constant. + + Checks ``init_map`` (pre-built from ``graph.initializer``) first, then searches + for a ``Constant`` node that produces ``name``. + """ + if name in init_map: + return init_map[name] + for node in graph.node: + if node.op_type != "Constant": + continue + if name in node.output: + for attr in node.attribute: + if attr.type == onnx.AttributeProto.TENSOR: + return numpy_helper.to_array(attr.t) + return None + + +def _read_implicit_gemm_attrs(node: onnx.NodeProto) -> Dict[str, object]: + out: Dict[str, object] = {} + for attr in node.attribute: + base = _normalize_attr(attr.name) + if attr.type == onnx.AttributeProto.INT: + out[base] = int(attr.i) + elif attr.type == onnx.AttributeProto.FLOAT: + out[base] = float(attr.f) + return out + + +def _replace_tensor_name(graph: onnx.GraphProto, old: str, new: str) -> None: + if old == new: + return + for n in graph.node: + for i, inp in enumerate(n.input): + if inp == old: + n.input[i] = new + for out in graph.output: + if out.name == old: + out.name = new + for vi in graph.value_info: + if vi.name == old: + vi.name = new + + +def _set_implicit_gemm_act_type(node: onnx.NodeProto, act_type: int) -> None: + kept = [a for a in node.attribute if _normalize_attr(a.name) != "act_type"] + del node.attribute[:] + node.attribute.extend(kept) + node.attribute.append(helper.make_attribute("act_type", int(act_type))) + + +def fuse_autoware_implicit_gemm_trailing_relu(model: onnx.ModelProto) -> int: + """Set ``act_type`` = kReLU (1) on ImplicitGemm and remove redundant ``Relu`` nodes.""" + + graph = model.graph + n_removed = 0 + + remove_idx: Set[int] = set() + + for ri, relu in enumerate(graph.node): + if ri in remove_idx: + continue + if relu.op_type != "Relu": + continue + if relu.domain not in ("", "ai.onnx"): + continue + if len(relu.input) < 1 or not relu.input[0]: + continue + if len(relu.output) < 1 or not relu.output[0]: + continue + + users: DefaultDict[str, List[int]] = defaultdict(list) + for ni, n in enumerate(graph.node): + if ni in remove_idx: + continue + for inp in n.input: + if inp: + users[inp].append(ni) + + g_out = relu.input[0] + r_out = relu.output[0] + + if len(users.get(g_out, [])) != 1: + continue + + producer_i: int | None = None + producer: onnx.NodeProto | None = None + for ni, n in enumerate(graph.node): + if g_out in n.output: + producer_i = ni + producer = n + break + if producer is None or producer_i is None: + continue + if producer.op_type != "ImplicitGemm" or producer.domain != "autoware": + continue + if producer_i in remove_idx: + continue + + attrs = _read_implicit_gemm_attrs(producer) + cur = int(attrs.get("act_type", 0) or 0) + if cur not in (0, 1): + continue + + _set_implicit_gemm_act_type(producer, 1) + _replace_tensor_name(graph, r_out, g_out) + remove_idx.add(ri) + n_removed += 1 + + if not remove_idx: + return 0 + + new_nodes: List[onnx.NodeProto] = [] + for ni, n in enumerate(graph.node): + if ni not in remove_idx: + new_nodes.append(n) + del graph.node[:] + graph.node.extend(new_nodes) + return n_removed diff --git a/deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py b/deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py new file mode 100644 index 000000000..a3584b43b --- /dev/null +++ b/deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py @@ -0,0 +1,295 @@ +"""FP32 sparse-encoder shadow for ``torch.onnx.export``. + +For split export the sparse tower is traced separately. This helper rebuilds a +fused FP32 ``BEVFusionSparseEncoder`` and copies float weights from the source +encoder, so BN can be folded (``fuse_spconv_bn``) into a clean sparse ONNX graph +without mutating the runtime model. +""" + +from __future__ import annotations + +import copy +import logging +from typing import Any, Dict, Mapping, Optional, Tuple + +import torch +import torch.nn as nn + +logger = logging.getLogger(__name__) + +# Attributes required to rebuild FP32 shadow encoder. +SPARSE_ENCODER_SHADOW_ATTRS: tuple[str, ...] = ( + "sparse_shape", + "in_channels", + "base_channels", + "output_channels", + "encoder_channels", + "encoder_paddings", + "num_aug_features", + "aug_features_min_values", + "aug_features_max_values", +) + + +def has_sparse_encoder_shadow_attributes(module: nn.Module) -> bool: + """True if ``module`` carries the config fields needed by ``build_float_sparse_encoder_shadow``.""" + return all(hasattr(module, name) for name in SPARSE_ENCODER_SHADOW_ATTRS) + + +def encoder_cfg_overrides_from_bevfusion_model(model: Optional[nn.Module]) -> Dict[str, Any]: + """Build shadow-attribute overrides from ``model.cfg.model['pts_middle_encoder']`` (MMEngine config).""" + if model is None: + return {} + cfg = getattr(model, "cfg", None) + if cfg is None: + return {} + model_dict = getattr(cfg, "model", None) + if model_dict is None: + return {} + try: + enc = model_dict.get("pts_middle_encoder") if hasattr(model_dict, "get") else None + except Exception: + return {} + if enc is None: + return {} + if isinstance(enc, Mapping) and not isinstance(enc, dict): + try: + enc = dict(enc) + except Exception: + return {} + if not isinstance(enc, dict): + return {} + out: Dict[str, Any] = {} + for name in SPARSE_ENCODER_SHADOW_ATTRS: + if name in enc: + out[name] = copy.deepcopy(enc[name]) + for name in ("norm_cfg", "block_type", "order", "return_middle_feats"): + if name in enc: + out[name] = copy.deepcopy(enc[name]) + return out + + +def resolve_sparse_onnx_shadow( + pts_middle_encoder: Optional[nn.Module], + bevfusion: Optional[nn.Module] = None, +) -> Tuple[Optional[nn.Module], Dict[str, Any]]: + """Pick sparse encoder source module and optional config overrides for shadow rebuild.""" + if pts_middle_encoder is None: + return None, {} + overrides = encoder_cfg_overrides_from_bevfusion_model(bevfusion) + if has_sparse_encoder_shadow_attributes(pts_middle_encoder): + return pts_middle_encoder, overrides + + can_fill = all(hasattr(pts_middle_encoder, name) or (name in overrides) for name in SPARSE_ENCODER_SHADOW_ATTRS) + if can_fill: + if overrides: + logger.info( + "Sparse ONNX shadow: merging %d key(s) from model.cfg pts_middle_encoder.", + len(overrides), + ) + return pts_middle_encoder, overrides + + logger.info( + "Sparse ONNX shadow: pts_middle_encoder lacks the config fields needed to rebuild an " + "FP32 shadow and model.cfg is incomplete; tracing the encoder directly." + ) + return None, {} + + +def build_float_sparse_encoder_shadow( + gm: nn.Module, + device: torch.device, + *, + cfg_overrides: Optional[Dict[str, Any]] = None, + fuse_spconv_bn: bool = True, +) -> nn.Module: + """Construct a fused FP32 ``BEVFusionSparseEncoder`` and load weights from ``gm`` state_dict. + + Only floating conv/BN parameters are copied from ``gm``. ``cfg_overrides`` supplies + fields missing on the source module (from ``model.cfg``). + """ + from mmengine.registry import MODELS, init_default_scope + + import projects.BEVFusion.bevfusion # noqa: F401 — register BEVFusionSparseEncoder + + init_default_scope("mmdet3d") + + def _pick(name: str) -> Any: + if cfg_overrides is not None and name in cfg_overrides: + return cfg_overrides[name] + return getattr(gm, name, None) + + missing = [r for r in SPARSE_ENCODER_SHADOW_ATTRS if _pick(r) is None] + if missing: + raise RuntimeError( + "Cannot rebuild FP32 sparse encoder for ONNX: source encoder + overrides missing: " + f"{missing}. Ensure the FP32 shadow encoder defines these (match training sparse_encoder), or pass " + f"a BEVFusion model with model.cfg.model.pts_middle_encoder." + ) + + def _buf_to_list(buf: torch.Tensor) -> list: + return buf.detach().cpu().flatten().tolist() + + def _aug_to_list(val: Any) -> list: + if isinstance(val, torch.Tensor): + return _buf_to_list(val) + if isinstance(val, (list, tuple)): + return list(val) + raise TypeError(f"aug_features_* must be tensor or list, got {type(val)!r}") + + default_norm = dict(type="BN1d", eps=1e-3, momentum=0.01) + nc = _pick("norm_cfg") + norm_cfg = copy.deepcopy(nc if nc is not None else default_norm) + + block_type = _pick("block_type") + if block_type is None: + block_type = "basicblock" + order_val = _pick("order") + order = tuple(order_val) if order_val is not None else ("conv", "norm", "act") + + enc_channels = _pick("encoder_channels") + if isinstance(enc_channels, torch.Tensor): + raise TypeError("encoder_channels must be nested tuples, not Tensor") + enc_paddings = _pick("encoder_paddings") + sparse_shape = _pick("sparse_shape") + sparse_shape = list(sparse_shape) if not isinstance(sparse_shape, list) else list(sparse_shape) + + ret_mid = _pick("return_middle_feats") + return_middle_feats = bool(ret_mid) if ret_mid is not None else False + + enc_cfg: Dict[str, Any] = dict( + type="BEVFusionSparseEncoder", + in_channels=int(_pick("in_channels")), + aug_features_min_values=_aug_to_list(_pick("aug_features_min_values")), + aug_features_max_values=_aug_to_list(_pick("aug_features_max_values")), + num_aug_features=int(_pick("num_aug_features")), + sparse_shape=sparse_shape, + order=order, + norm_cfg=norm_cfg, + base_channels=int(_pick("base_channels")), + output_channels=int(_pick("output_channels")), + encoder_channels=enc_channels, + encoder_paddings=enc_paddings, + block_type=block_type, + return_middle_feats=return_middle_feats, + ) + + enc: nn.Module = MODELS.build(enc_cfg) + enc.to(device) + enc.eval() + + if fuse_spconv_bn: + from deployment.projects.bevfusion.export.spconv_bn_fusion import fuse_spconv_bn_in_encoder + + fuse_spconv_bn_in_encoder(enc) + else: + logger.info("Sparse ONNX float shadow: keep SparseConv+BN unfused (fuse_spconv_bn=False).") + + gm_sd = gm.state_dict() + enc_sd = enc.state_dict() + + def _align_5d_spconv_weight_to_krsc(v: torch.Tensor, target: torch.Size) -> Optional[torch.Tensor]: + """Some checkpoints store 5D sparse conv as (C_in, C_out, Kz, Ky, Kx); MMDet encoder uses KRSC (C_out, Kz, Ky, Kx, C_in).""" + if v.dim() != 5 or len(target) != 5: + return None + if v.shape == target: + return v + # Explicit ICOC -> KRSC when channel/spatial layout matches. + if ( + v.shape[0] == target[4] + and v.shape[1] == target[0] + and v.shape[2] == target[1] + and v.shape[3] == target[2] + and v.shape[4] == target[3] + ): + return v.permute(1, 2, 3, 4, 0).contiguous() + perm = v.permute(1, 2, 3, 4, 0).contiguous() + if perm.shape == target: + return perm + perm2 = v.permute(4, 0, 1, 2, 3).contiguous() + if perm2.shape == target: + return perm2 + return None + + def _flat_state_key(key: str) -> str: + """Legacy checkpoints may use underscore keys (e.g. ``encoder_layers_encoder_layer1_0_conv1``).""" + return key.replace(".", "_") + + def _gm_value_for_key(key: str) -> Optional[torch.Tensor]: + flat = _flat_state_key(key) + for cand in ( + key, + f"module.{key}", + f"pts_middle_encoder.{key}", + flat, + f"module.{flat}", + f"pts_middle_encoder.{flat}", + ): + if cand in gm_sd: + return gm_sd[cand] # type: ignore[return-value] + if key.startswith("module.") and key[len("module.") :] in gm_sd: + return gm_sd[key[len("module.") :]] # type: ignore[return-value] + return None + + # Copy with ``Tensor.copy_`` instead of ``load_state_dict``: spconv registers + # ``load_state_dict`` pre-hooks that permute *disk* layouts when SPCONV_SAVED_WEIGHT_LAYOUT + # is set; mutating the same dict we validated can also desync shapes vs. plain Parameters. + n_copied = 0 + with torch.no_grad(): + for k, t in enc_sd.items(): + v = _gm_value_for_key(k) + if v is None or not torch.is_tensor(v): + continue + + v = v.detach() + if v.dim() == 5 and t.dim() == 5 and v.shape != t.shape: + aligned = _align_5d_spconv_weight_to_krsc(v, t.shape) + if aligned is not None: + logger.info( + "Float shadow ICOC->KRSC %s: %s -> %s", + k, + tuple(v.shape), + tuple(aligned.shape), + ) + v = aligned + else: + logger.debug("Float shadow skip 5D %s: gm %s vs enc %s", k, tuple(v.shape), tuple(t.shape)) + continue + elif v.shape != t.shape: + continue + + if v.dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64): + w = v.to(device=t.device, dtype=t.dtype, non_blocking=False).contiguous() + elif v.dtype in (torch.int32, torch.int64, torch.bool): + w = v.to(device=t.device, non_blocking=False).contiguous() + else: + continue + + if w.shape != t.shape: + raise RuntimeError( + f"Float shadow internal error: key {k!r} tensor shape {tuple(w.shape)} vs encoder " + f"{tuple(t.shape)} after layout fix." + ) + + parent_path, dot, leaf = k.rpartition(".") + if not dot: + continue + try: + sub = enc.get_submodule(parent_path) + except AttributeError: + logger.debug("Float shadow: no submodule for state key %s", k) + continue + dst = getattr(sub, leaf, None) + if dst is None or not torch.is_tensor(dst): + continue + dst.copy_(w) + n_copied += 1 + + logger.info( + "Sparse ONNX float shadow: copied %d / %d state entries from source encoder via in-place copy " + "(bypasses spconv load_state_dict hooks).", + n_copied, + len(enc_sd), + ) + + return enc diff --git a/deployment/projects/bevfusion/export/spconv_bn_fusion.py b/deployment/projects/bevfusion/export/spconv_bn_fusion.py new file mode 100644 index 000000000..19dcd7e77 --- /dev/null +++ b/deployment/projects/bevfusion/export/spconv_bn_fusion.py @@ -0,0 +1,46 @@ +"""Fuse ``SparseConvolution + BatchNorm1d`` in a BEVFusion sparse encoder. + +Plain graph optimization used by the FP16 deployment export path: fold ``Conv -> BN`` +into the conv weights (eval mode) so the exported sparse ONNX is BN-free and matches the +runtime graph. This is *not* quantization — it only rewrites the module tree. +""" + +from __future__ import annotations + +import logging + +import torch +import torch.nn as nn + +logger = logging.getLogger(__name__) + + +def fuse_spconv_bn_in_encoder(sparse_encoder: nn.Module) -> int: + """Fuse each ``SparseConvolution`` + ``BatchNorm1d`` pair in ``sparse_encoder`` (eval mode). + + Returns the number of fused Conv-BN pairs. + """ + try: + # spconv ships an eval-mode Conv+BN fold helper (library utility, not INT8). + from spconv.pytorch.quantization.utils import fuse_spconv_bn_eval + except ImportError: + logger.warning("spconv BN-fusion helper not available; skipping SparseConv+BN fusion") + return 0 + + from spconv.pytorch.conv import SparseConvolution + + sparse_encoder.eval() + fused_count = 0 + + for module in sparse_encoder.modules(): + children = list(module._modules.items()) + for i in range(len(children) - 1): + left_name, left_mod = children[i] + right_name, right_mod = children[i + 1] + if isinstance(left_mod, SparseConvolution) and isinstance(right_mod, torch.nn.BatchNorm1d): + fused_conv = fuse_spconv_bn_eval(left_mod, right_mod) + setattr(module, left_name, fused_conv) + setattr(module, right_name, torch.nn.Identity()) + fused_count += 1 + + return fused_count diff --git a/deployment/projects/bevfusion/export/tensorrt_export_pipeline.py b/deployment/projects/bevfusion/export/tensorrt_export_pipeline.py new file mode 100644 index 000000000..c9d299c85 --- /dev/null +++ b/deployment/projects/bevfusion/export/tensorrt_export_pipeline.py @@ -0,0 +1,136 @@ +"""BEVFusion TensorRT export pipeline. + +Converts BEVFusion ONNX (single or split sparse+dense) to TensorRT engine(s). +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Dict, Optional, Tuple + +import torch + +from deployment.config.base import BaseDeploymentConfig +from deployment.config.schema import ComponentsConfig +from deployment.export.exporters.tensorrt_exporter import TensorRTExporter +from deployment.primitives.artifacts import Artifact +from deployment.primitives.device import DeviceSpec +from deployment.primitives.tensorrt_plugins import load_tensorrt_plugin_libraries +from deployment.projects.bevfusion.io.component_utils import is_split_bevfusion_components + + +class BEVFusionTensorRTExportPipeline: + """TensorRT export for BEVFusion (one engine or sparse+dense pair).""" + + def __init__( + self, + components_cfg: ComponentsConfig, + plugin_libraries: Tuple[str, ...] = (), + logger: Optional[logging.Logger] = None, + ) -> None: + self._components_cfg = components_cfg + self._plugin_libraries = plugin_libraries + self.logger = logger or logging.getLogger(__name__) + + def export( + self, + *, + onnx_path: str, + output_dir: str, + config: BaseDeploymentConfig, + device: DeviceSpec, + ) -> Artifact: + if not device.is_cuda: + raise ValueError(f"TensorRT export requires CUDA device, got: {device}") + + torch.cuda.set_device(device.index) + + # Load any custom plugin .so libraries (e.g. the BEVFusion spconv ImplicitGemm plugin) + # before building. No-op when plugin_libraries is empty. + load_tensorrt_plugin_libraries(self.logger, getattr(self, "_plugin_libraries", ())) + + onnx_dir = Path(onnx_path) + output_dir_path = Path(output_dir) + output_dir_path.mkdir(parents=True, exist_ok=True) + + if is_split_bevfusion_components(self._components_cfg): + return self._export_split_engines(onnx_dir, output_dir_path, config) + + component_cfg = self._components_cfg.get_component("bevfusion_main_body") + onnx_file = onnx_dir / component_cfg.onnx_file + engine_file = output_dir_path / component_cfg.engine_file + + if not onnx_file.exists(): + raise FileNotFoundError(f"ONNX file not found: {onnx_file}") + + self.logger.info("=" * 80) + self.logger.info("Converting BEVFusion ONNX to TensorRT") + self.logger.info("=" * 80) + self.logger.info(f"ONNX: {onnx_file}") + self.logger.info(f"Engine: {engine_file}") + + artifact = TensorRTExporter(config=config.get_tensorrt_settings("bevfusion_main_body")).export( + onnx_path=str(onnx_file), + output_path=str(engine_file), + ) + + self.logger.info(f"TensorRT engine saved: {artifact.path}") + self.logger.info("=" * 80) + + return Artifact(path=str(output_dir_path)) + + def _export_split_engines( + self, + onnx_dir: Path, + output_dir_path: Path, + config: BaseDeploymentConfig, + ) -> Artifact: + if not onnx_dir.is_dir(): + raise ValueError(f"Split TensorRT export expects ONNX directory, got: {onnx_dir}") + + onnx_files = sorted( + (path for path in onnx_dir.iterdir() if path.is_file() and path.suffix.lower() == ".onnx"), + key=lambda p: p.name, + ) + if not onnx_files: + raise FileNotFoundError(f"No ONNX files in {onnx_dir}") + + engine_file_map = self._build_engine_file_map() + onnx_stem_to_component = self._build_onnx_stem_to_component_map() + + self.logger.info("=" * 80) + self.logger.info("Converting split BEVFusion ONNX → TensorRT (sparse + dense)") + self.logger.info("=" * 80) + + for i, onnx_file in enumerate(onnx_files, 1): + onnx_stem = onnx_file.stem + if onnx_stem not in engine_file_map: + raise KeyError(f"ONNX file '{onnx_file.name}' is not declared in deploy config components.*.onnx_file") + engine_file = engine_file_map[onnx_stem] + trt_path = output_dir_path / engine_file + trt_path.parent.mkdir(parents=True, exist_ok=True) + + component_name = onnx_stem_to_component[onnx_stem] + self.logger.info("[%d/%d] %s → %s", i, len(onnx_files), onnx_file.name, trt_path.name) + + TensorRTExporter(config=config.get_tensorrt_settings(component_name)).export( + onnx_path=str(onnx_file), + output_path=str(trt_path), + ) + + self.logger.info("Split TensorRT engines written to %s", output_dir_path) + self.logger.info("=" * 80) + return Artifact(path=str(output_dir_path)) + + def _build_engine_file_map(self) -> Dict[str, str]: + mapping: Dict[str, str] = {} + for _name, comp in self._components_cfg.items(): + mapping[Path(comp.onnx_file).stem] = comp.engine_file + return mapping + + def _build_onnx_stem_to_component_map(self) -> Dict[str, str]: + return { + Path(component_cfg.onnx_file).stem: component_name + for component_name, component_cfg in self._components_cfg.items() + } diff --git a/deployment/projects/bevfusion/inference/__init__.py b/deployment/projects/bevfusion/inference/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py b/deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py new file mode 100644 index 000000000..b029ac940 --- /dev/null +++ b/deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py @@ -0,0 +1,310 @@ +"""BEVFusion Deployment Pipeline Base Class. + +Provides common preprocessing, postprocessing, and inference logic +shared by PyTorch, ONNX, and TensorRT backend implementations. +""" + +from __future__ import annotations + +import logging +import os +import time +from abc import abstractmethod +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.primitives.device import DeviceSpec + +logger = logging.getLogger(__name__) + +_DEBUG_POSTPROCESS = os.environ.get("BEVFUSION_DEBUG_POSTPROCESS", "").strip().lower() in ("1", "true", "yes") +_DEBUG_POSTPROCESS_MAX = 2 +_postprocess_debug_count = 0 + + +def _env_int_pp(key: str, default: int) -> int: + try: + return int(os.environ.get(key, str(default)).strip()) + except ValueError: + return default + + +class BEVFusionDeploymentPipeline(BaseInferencePipeline): + """Base pipeline for BEVFusion inference. + + Handles voxelization in preprocessing and bbox decoding in postprocessing. + The model (ONNX/TensorRT) takes voxels/coors/num_points_per_voxel and + outputs bbox_pred/score/label_pred directly. + """ + + def __init__( + self, + pytorch_model: torch.nn.Module, + backend_type: Backend, + device: DeviceSpec, + ) -> None: + cfg = getattr(pytorch_model, "cfg", None) + + class_names = getattr(cfg, "class_names", None) + point_cloud_range = getattr(cfg, "point_cloud_range", None) + voxel_size = getattr(cfg, "voxel_size", None) + + if class_names is None: + raise ValueError("class_names not found in pytorch_model.cfg") + + super().__init__( + model=pytorch_model, + backend_type=backend_type, + device=device, + ) + + self.pytorch_model: torch.nn.Module = pytorch_model + self.num_classes: int = len(class_names) + self.class_names: List[str] = class_names + self.point_cloud_range: Optional[List[float]] = point_cloud_range + self.voxel_size: Optional[List[float]] = voxel_size + + def to_device_tensor(self, data: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + if isinstance(data, np.ndarray): + data = torch.from_numpy(data) + return data.to(self.torch_device) + + def to_numpy(self, data: torch.Tensor, dtype: np.dtype = np.float32) -> np.ndarray: + arr = data.cpu().numpy().astype(dtype) + if not arr.flags["C_CONTIGUOUS"]: + arr = np.ascontiguousarray(arr) + return arr + + @override + def preprocess( + self, + points: torch.Tensor, + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, object]]: + """Voxelize point cloud into voxels/coors/num_points_per_voxel. + + Uses the BEVFusion model's voxelization layer (outside the ONNX graph). + + Args: + points: Point cloud tensor [N, point_features]. + + Returns: + Tuple of (preprocessed_dict, metadata_dict). + """ + points_tensor = self.to_device_tensor(points).float() + + with torch.no_grad(): + feats, coords, sizes = [], [], [] + ret = self.pytorch_model.pts_voxel_layer(points_tensor) + if len(ret) == 3: + f, c, n = ret + else: + f, c = ret + n = None + feats.append(f) + coords.append(c) + if n is not None: + sizes.append(n) + + voxels = torch.cat(feats, dim=0) + coors = torch.cat(coords, dim=0) + num_points_per_voxel = ( + torch.cat(sizes, dim=0) if sizes else torch.ones(voxels.shape[0], device=voxels.device) + ) + + preprocessed_dict = { + "voxels": voxels, + "coors": coors, + "num_points_per_voxel": num_points_per_voxel, + } + return preprocessed_dict, {} + + @override + def run_model( + self, + preprocessed_input: Dict[str, torch.Tensor], + ) -> Tuple[List[torch.Tensor], Dict[str, float]]: + """Run the BEVFusion model and return raw outputs with latency. + + Args: + preprocessed_input: Dict with voxels, coors, num_points_per_voxel. + + Returns: + Tuple of ([bbox_pred, score, label_pred], stage_latencies). + """ + stage_latencies: Dict[str, float] = {} + + start = time.perf_counter() + outputs = self.run_bevfusion( + preprocessed_input["voxels"], + preprocessed_input["coors"], + preprocessed_input["num_points_per_voxel"], + ) + stage_latencies["bevfusion_ms"] = (time.perf_counter() - start) * 1000 + + return outputs, stage_latencies + + @override + def postprocess( + self, + model_outputs: List[torch.Tensor], + sample_meta: Dict[str, object], + ) -> List[Dict[str, Union[List[float], float, int]]]: + """Decode bbox_pred/score/label_pred into detection dicts. + + The BEVFusion ONNX model already includes query scoring / selection, but + bbox outputs are still in the head-encoded space and must be decoded to + metric coordinates: + - bbox_pred: [10, num_proposals] + (center_x_feat, center_y_feat, z_gravity, dim0_log, dim1_log, dim2_log, sin, cos, vx, vy) + - score: [num_proposals] + - label_pred: [num_proposals] + + Args: + model_outputs: [bbox_pred, score, label_pred] tensors. + sample_meta: Sample metadata. + + Returns: + List of detection dicts with bbox_3d, score, label. + """ + bbox_pred, score, label_pred = [self.to_device_tensor(o) for o in model_outputs] + + # Normalize common export/runtime shapes to [10, num_proposals], [num_proposals], [num_proposals]. + if bbox_pred.ndim == 3 and bbox_pred.shape[0] == 1: + bbox_pred = bbox_pred[0] + if bbox_pred.ndim == 2 and bbox_pred.shape[0] != 10 and bbox_pred.shape[1] == 10: + bbox_pred = bbox_pred.transpose(0, 1).contiguous() + if bbox_pred.ndim != 2 or bbox_pred.shape[0] != 10: + logger.warning(f"Unexpected bbox_pred shape {tuple(bbox_pred.shape)}; skipping frame.") + return [] + + score = score.reshape(-1) + label_pred = label_pred.reshape(-1) + + num_proposals = min(bbox_pred.shape[1], score.shape[0], label_pred.shape[0]) + if num_proposals == 0: + return [] + + global _postprocess_debug_count + dbg_max = max(0, _env_int_pp("BEVFUSION_DEBUG_POSTPROCESS_FRAMES", _DEBUG_POSTPROCESS_MAX)) + did_pp_dbg = False + if _DEBUG_POSTPROCESS and _postprocess_debug_count < dbg_max: + _postprocess_debug_count += 1 + did_pp_dbg = True + sc = score[:num_proposals].float() + lb = label_pred[:num_proposals].long() + cx = bbox_pred[0, :num_proposals].float() + cy = bbox_pred[1, :num_proposals].float() + uniq_l = torch.unique(lb) + logger.warning( + "[debug-postprocess] frame=%d/%d backend=%s num_proposals=%d " + "score[min,max,mean]=[%.6f,%.6f,%.6f] score>0.1:%d score>0.5:%d " + "label[min,max]=[%d,%d] label_unique=%s " + "center_feat_x[min,max]=[%.4f,%.4f] center_feat_y[min,max]=[%.4f,%.4f]", + _postprocess_debug_count, + dbg_max, + str(self.backend_type), + int(num_proposals), + float(sc.min().item()), + float(sc.max().item()), + float(sc.mean().item()), + int((sc > 0.1).sum().item()), + int((sc > 0.5).sum().item()), + int(lb.min().item()), + int(lb.max().item()), + str(uniq_l.detach().cpu().tolist()), + float(cx.min().item()), + float(cx.max().item()), + float(cy.min().item()), + float(cy.max().item()), + ) + + # Decode via BEVFusion's own bbox_coder to avoid convention drift. + bbox_coder = getattr(self.pytorch_model.bbox_head, "bbox_coder", None) + if bbox_coder is None: + logger.warning("bbox_coder not found on model.bbox_head; skipping frame.") + return [] + + center = bbox_pred[0:2, :num_proposals].unsqueeze(0) + height = bbox_pred[2:3, :num_proposals].unsqueeze(0) + dim = bbox_pred[3:6, :num_proposals].unsqueeze(0) + rot = bbox_pred[6:8, :num_proposals].unsqueeze(0) + vel = bbox_pred[8:10, :num_proposals].unsqueeze(0) + + labels = label_pred[:num_proposals].long() + scores = score[:num_proposals].to(dtype=bbox_pred.dtype) + heatmap = torch.zeros((1, self.num_classes, num_proposals), device=self.torch_device, dtype=bbox_pred.dtype) + valid = (labels >= 0) & (labels < self.num_classes) + if valid.any(): + valid_idx = torch.nonzero(valid, as_tuple=False).reshape(-1) + heatmap[0, labels[valid_idx], valid_idx] = scores[valid_idx] + + decoded = bbox_coder.decode(heatmap, rot, dim, center, height, vel, filter=False)[0] + decoded_boxes = decoded["bboxes"] + decoded_scores = decoded["scores"] + decoded_labels = decoded["labels"] + + results: List[Dict[str, Union[List[float], float, int]]] = [] + for i in range(decoded_boxes.shape[0]): + s = float(decoded_scores[i].item()) + if s < 1e-6: + continue + + bbox = decoded_boxes[i].detach().cpu().numpy() + # decoded box format: [x, y, z, dx, dy, dz, yaw, vx, vy] + if bbox.shape[0] < 7: + continue + + cx, cy, z = float(bbox[0]), float(bbox[1]), float(bbox[2]) + d0, d1, d2 = float(bbox[3]), float(bbox[4]), float(bbox[5]) + yaw = float(bbox[6]) + vx = float(bbox[7]) if bbox.shape[0] > 7 else 0.0 + vy = float(bbox[8]) if bbox.shape[0] > 8 else 0.0 + + results.append( + { + "bbox_3d": [cx, cy, z, d0, d1, d2, yaw, vx, vy], + "score": s, + "label": int(decoded_labels[i].item()), + } + ) + + if did_pp_dbg and decoded_boxes.shape[0] > 0: + b = decoded_boxes[:, :3].detach().float() + logger.warning( + "[debug-postprocess] decoded metric centers N=%d " + "x[min,max]=[%.2f,%.2f] y[min,max]=[%.2f,%.2f] z[min,max]=[%.2f,%.2f] " + "(compare to point_cloud_range / GT — wild ranges → mAP 0 with many preds)", + int(decoded_boxes.shape[0]), + float(b[:, 0].min().item()), + float(b[:, 0].max().item()), + float(b[:, 1].min().item()), + float(b[:, 1].max().item()), + float(b[:, 2].min().item()), + float(b[:, 2].max().item()), + ) + + return results + + @abstractmethod + def run_bevfusion( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> List[torch.Tensor]: + """Run the BEVFusion model. + + Args: + voxels: [M, max_points, C] + coors: [M, 3] (z, y, x) + num_points_per_voxel: [M] + + Returns: + [bbox_pred, score, label_pred] + """ + raise NotImplementedError diff --git a/deployment/projects/bevfusion/inference/onnx_inference_pipeline.py b/deployment/projects/bevfusion/inference/onnx_inference_pipeline.py new file mode 100644 index 000000000..99d184200 --- /dev/null +++ b/deployment/projects/bevfusion/inference/onnx_inference_pipeline.py @@ -0,0 +1,165 @@ +"""BEVFusion ONNX Pipeline Implementation.""" + +from __future__ import annotations + +import logging +import os.path as osp +from typing import List, Optional + +import numpy as np +import onnxruntime as ort +import torch +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.config.schema import ComponentsConfig +from deployment.primitives.artifacts import resolve_artifact_path +from deployment.primitives.device import DeviceSpec +from deployment.projects.bevfusion.inference.bevfusion_inference_pipeline import BEVFusionDeploymentPipeline +from deployment.projects.bevfusion.io.component_utils import has_component, is_split_bevfusion_components +from deployment.projects.bevfusion.io.coors_contract import voxel_indices_xyz_to_graph_input_zyx + +logger = logging.getLogger(__name__) + + +class BEVFusionONNXPipeline(BEVFusionDeploymentPipeline): + """ONNXRuntime-based BEVFusion pipeline. + + Single ONNX: voxels/coors/num_points → bbox_pred/score/label_pred. + + Split ONNX: sparse session → ``lidar_bev``, then dense session → outputs. + """ + + def __init__( + self, + pytorch_model: torch.nn.Module, + onnx_dir: str, + device: DeviceSpec, + components_cfg: ComponentsConfig, + ) -> None: + super().__init__(pytorch_model=pytorch_model, backend_type=Backend.ONNX, device=device) + + self.onnx_dir = onnx_dir + self._components_cfg = components_cfg + split_layout = is_split_bevfusion_components(components_cfg) + merged_model_available = False + if split_layout and has_component(components_cfg, "bevfusion_main_body"): + merged_path = resolve_artifact_path( + base_dir=self.onnx_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_main_body", + file_key="onnx_file", + ) + merged_model_available = osp.exists(merged_path) + self._split = split_layout and not merged_model_available + self.session: Optional[ort.InferenceSession] = None + self._session_sparse: Optional[ort.InferenceSession] = None + self._session_dense: Optional[ort.InferenceSession] = None + self._load_onnx_model() + logger.info(f"BEVFusion ONNX pipeline initialized from: {onnx_dir} (split={self._split})") + + def _load_onnx_model(self) -> None: + so = ort.SessionOptions() + so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + providers = self.device.to_ort_provider() + + if self._split: + sparse_path = resolve_artifact_path( + base_dir=self.onnx_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_sparse", + file_key="onnx_file", + ) + dense_path = resolve_artifact_path( + base_dir=self.onnx_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_dense", + file_key="onnx_file", + ) + if not osp.exists(sparse_path): + raise FileNotFoundError(f"Sparse ONNX not found: {sparse_path}") + if not osp.exists(dense_path): + raise FileNotFoundError(f"Dense ONNX not found: {dense_path}") + self._session_sparse = ort.InferenceSession(sparse_path, sess_options=so, providers=providers) + self._session_dense = ort.InferenceSession(dense_path, sess_options=so, providers=providers) + logger.info("Loaded split ONNX: %s , %s", sparse_path, dense_path) + return + + model_path = resolve_artifact_path( + base_dir=self.onnx_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_main_body", + file_key="onnx_file", + ) + if not osp.exists(model_path): + raise FileNotFoundError(f"BEVFusion ONNX not found: {model_path}") + + self.session = ort.InferenceSession(model_path, sess_options=so, providers=providers) + logger.info(f"Loaded BEVFusion ONNX: {model_path}") + + @override + def run_bevfusion( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> List[torch.Tensor]: + if self._split: + return self._run_bevfusion_split(voxels, coors, num_points_per_voxel) + + assert self.session is not None + voxels_np = self.to_numpy(voxels, dtype=np.float32) + coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) + num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) + + input_names = [inp.name for inp in self.session.get_inputs()] + output_names = [out.name for out in self.session.get_outputs()] + + feed_dict = {} + for name in input_names: + if "voxel" in name.lower() and "num" not in name.lower(): + feed_dict[name] = voxels_np + elif "coor" in name.lower(): + feed_dict[name] = coors_np + elif "num" in name.lower(): + feed_dict[name] = num_points_np + + outputs = self.session.run(output_names, feed_dict) + return [torch.from_numpy(out).to(self.torch_device) for out in outputs] + + def _run_bevfusion_split( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> List[torch.Tensor]: + assert self._session_sparse is not None and self._session_dense is not None + + voxels_np = self.to_numpy(voxels, dtype=np.float32) + coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) + num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) + + s_in = [inp.name for inp in self._session_sparse.get_inputs()] + s_out = [out.name for out in self._session_sparse.get_outputs()] + + sparse_feed = {} + for name in s_in: + ln = name.lower() + if "voxel" in ln and "num" not in ln: + sparse_feed[name] = voxels_np + elif "coor" in ln: + sparse_feed[name] = coors_np + elif "num" in ln: + sparse_feed[name] = num_points_np + + sparse_ort_outs = self._session_sparse.run(s_out, sparse_feed) + if len(sparse_ort_outs) != 1: + raise RuntimeError(f"Expected 1 sparse output, got {len(sparse_ort_outs)}") + lidar_bev_np = np.ascontiguousarray(sparse_ort_outs[0].astype(np.float32)) + + d_in = [inp.name for inp in self._session_dense.get_inputs()] + d_out = [out.name for out in self._session_dense.get_outputs()] + dense_feed = {d_in[0]: lidar_bev_np} + + dense_ort_outs = self._session_dense.run(d_out, dense_feed) + return [torch.from_numpy(out).to(self.torch_device) for out in dense_ort_outs] diff --git a/deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py b/deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py new file mode 100644 index 000000000..7827b05cd --- /dev/null +++ b/deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py @@ -0,0 +1,234 @@ +"""BEVFusion PyTorch Pipeline Implementation with per-block latency.""" + +from __future__ import annotations + +import logging +import time +from typing import Dict, List, Tuple + +import torch +import torch.nn.functional as F +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.primitives.device import DeviceSpec +from deployment.projects.bevfusion.inference.bevfusion_inference_pipeline import BEVFusionDeploymentPipeline + +try: + from projects.BEVFusion.bevfusion.bevfusion import _ensure_float_for_pts_pipeline as _ensure_float_for_pts_impl +except Exception: + _ensure_float_for_pts_impl = None + +logger = logging.getLogger(__name__) + + +_PYTORCH_TENSOR_LOG_PREFIX = "[BEVFUSION][PyTorch][tensors]" + + +def _ensure_float_for_pts_pipeline(tensor: torch.Tensor) -> torch.Tensor: + """Best-effort compatibility wrapper for BEVFusion sparse feature dtype normalization.""" + if _ensure_float_for_pts_impl is not None: + return _ensure_float_for_pts_impl(tensor) + return tensor.float() if tensor.dtype != torch.float32 else tensor + + +def _tensor_stats(t: torch.Tensor, name: str) -> str: + """Return a compact string with tensor statistics for debugging.""" + t_f = t.float() + return ( + f"{_PYTORCH_TENSOR_LOG_PREFIX} {name}: shape={tuple(t.shape)} dtype={t.dtype} " + f"min={t_f.min().item():.4f} max={t_f.max().item():.4f} " + f"mean={t_f.mean().item():.4f} std={t_f.std().item():.4f} " + f"abs_mean={t_f.abs().mean().item():.4f} " + f"nonzero={t_f.count_nonzero().item()}/{t_f.numel()}" + ) + + +class BEVFusionPyTorchPipeline(BEVFusionDeploymentPipeline): + """PyTorch-based BEVFusion pipeline with per-block latency breakdown. + + Runs the full model natively, structured to match the ONNX/TensorRT + staged inference for output consistency. Reports latency for each block: + - voxel_encoder_ms: voxel mean reduction + - sparse_encoder_ms: pts_middle_encoder (spconv) + - backbone_ms: pts_backbone (SECOND) + - neck_ms: pts_neck (SECONDFPN) + - head_ms: bbox_head + postprocess scoring + """ + + _debug_frame_count = 0 + + def __init__(self, pytorch_model: torch.nn.Module, device: DeviceSpec) -> None: + super().__init__(pytorch_model=pytorch_model, backend_type=Backend.PYTORCH, device=device) + logger.info("BEVFusion PyTorch pipeline initialized (per-block latency enabled)") + + @override + def run_model( + self, + preprocessed_input: Dict[str, torch.Tensor], + ) -> Tuple[List[torch.Tensor], Dict[str, float]]: + """Run BEVFusion with per-block latency measurement. + + Breaks the model into stages and measures each one independently. + """ + stage_latencies: Dict[str, float] = {} + + total_start = time.perf_counter() + outputs = self._run_bevfusion_with_breakdown( + preprocessed_input["voxels"], + preprocessed_input["coors"], + preprocessed_input["num_points_per_voxel"], + stage_latencies, + ) + stage_latencies["bevfusion_ms"] = (time.perf_counter() - total_start) * 1000 + + return outputs, stage_latencies + + def _run_bevfusion_with_breakdown( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + stage_latencies: Dict[str, float], + ) -> List[torch.Tensor]: + """Run BEVFusion stage by stage, collecting per-block latencies.""" + model = self.pytorch_model + model.eval() + device = self.torch_device + + voxels = voxels.to(device) + coors = coors.to(device) + num_points_per_voxel = num_points_per_voxel.to(device) + + with torch.no_grad(): + # --- Stage 1: Voxel Encoder (mean reduction) --- + torch.cuda.synchronize() + t0 = time.perf_counter() + + if coors.shape[1] == 3: + num_points = coors.shape[0] + batch_coors = torch.zeros(num_points, 1, device=device, dtype=coors.dtype) + coors = torch.cat([batch_coors, coors], dim=1).contiguous() + + if getattr(model, "voxelize_reduce", True): + npt = num_points_per_voxel.type_as(voxels).view(-1, 1).clamp(min=1.0) + voxel_features = voxels.sum(dim=1, keepdim=False) / npt + else: + voxel_features = voxels + + torch.cuda.synchronize() + stage_latencies["voxel_encoder_ms"] = (time.perf_counter() - t0) * 1000 + + # --- Stage 2: Sparse Encoder (pts_middle_encoder / spconv) --- + torch.cuda.synchronize() + t1 = time.perf_counter() + + _dbg = BEVFusionPyTorchPipeline._debug_frame_count < 2 + BEVFusionPyTorchPipeline._debug_frame_count += 1 + if _dbg: + print( + f"{_PYTORCH_TENSOR_LOG_PREFIX} frame={BEVFusionPyTorchPipeline._debug_frame_count}/2 " + f"(native pts_middle_encoder → backbone → neck → head)" + ) + print(_tensor_stats(voxel_features, "voxel_features_input")) + + spatial_features = model.pts_middle_encoder(voxel_features, coors, batch_size=1) + spatial_features = _ensure_float_for_pts_pipeline(spatial_features) + + if _dbg: + print(_tensor_stats(spatial_features, "sparse_encoder_output")) + + torch.cuda.synchronize() + stage_latencies["sparse_encoder_ms"] = (time.perf_counter() - t1) * 1000 + + # --- Stage 3: Backbone (pts_backbone / SECOND) --- + torch.cuda.synchronize() + t2 = time.perf_counter() + + backbone_out = spatial_features + if hasattr(model, "pts_backbone") and model.pts_backbone is not None: + backbone_out = model.pts_backbone(_ensure_float_for_pts_pipeline(spatial_features)) + + if _dbg: + if isinstance(backbone_out, (list, tuple)): + for bi, bo in enumerate(backbone_out): + print(_tensor_stats(bo, f"backbone_out[{bi}]")) + else: + print(_tensor_stats(backbone_out, "backbone_out")) + + torch.cuda.synchronize() + stage_latencies["backbone_ms"] = (time.perf_counter() - t2) * 1000 + + # --- Stage 4: Neck (pts_neck / SECONDFPN) --- + torch.cuda.synchronize() + t3 = time.perf_counter() + + neck_out = backbone_out + if hasattr(model, "pts_neck") and model.pts_neck is not None: + neck_out = model.pts_neck(backbone_out) + + # Match ``BEVFusion.extract_feat``: head ``bev_pos`` is built for + # ``grid_size // out_size_factor`` (e.g. 180×180) while SECOND/FPN can + # yield full voxel BEV (e.g. 1440×1440). Skipping this pools causes + # ``key`` vs ``key_pos`` length mismatch in the transformer decoder. + align_fn = getattr(model, "_align_lidar_bev_to_head_grid", None) + if callable(align_fn): + neck_out = align_fn(neck_out) + + if _dbg: + if isinstance(neck_out, (list, tuple)): + for ni, no in enumerate(neck_out): + print(_tensor_stats(no, f"neck_out[{ni}]")) + else: + print(_tensor_stats(neck_out, "neck_out")) + + torch.cuda.synchronize() + stage_latencies["neck_ms"] = (time.perf_counter() - t3) * 1000 + + # --- Stage 5: Detection Head (bbox_head) --- + torch.cuda.synchronize() + t4 = time.perf_counter() + + preds = model.bbox_head(neck_out, []) + + torch.cuda.synchronize() + stage_latencies["head_ms"] = (time.perf_counter() - t4) * 1000 + + # --- Stage 6: Post-scoring --- + torch.cuda.synchronize() + t5 = time.perf_counter() + + preds = preds[0][0] + + if _dbg: + print(_tensor_stats(preds["heatmap"], "head_heatmap_raw")) + print(_tensor_stats(preds["center"][0], "head_center")) + print(_tensor_stats(preds["dim"][0], "head_dim")) + + score = preds["heatmap"].sigmoid() + one_hot = F.one_hot(preds["query_labels"], num_classes=score.size(1)).permute(0, 2, 1) + score = score * preds["query_heatmap_score"] * one_hot + score = score[0].max(dim=0)[0] + + bbox_pred = torch.cat( + [preds["center"][0], preds["height"][0], preds["dim"][0], preds["rot"][0], preds["vel"][0]], + dim=0, + ) + label_pred = preds["query_labels"][0] + + torch.cuda.synchronize() + stage_latencies["post_scoring_ms"] = (time.perf_counter() - t5) * 1000 + + return [bbox_pred, score, label_pred] + + @override + def run_bevfusion( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> List[torch.Tensor]: + # Not used: this pipeline overrides run_model() to call _run_bevfusion_with_breakdown() + # directly, so the base template never dispatches through run_bevfusion(). Present only to + # satisfy the abstract method on BEVFusionDeploymentPipeline. + raise NotImplementedError("BEVFusionPyTorchPipeline overrides run_model(); run_bevfusion() is not used.") diff --git a/deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py b/deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py new file mode 100644 index 000000000..9f85b1481 --- /dev/null +++ b/deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py @@ -0,0 +1,808 @@ +"""BEVFusion TensorRT Pipeline Implementation.""" + +from __future__ import annotations + +import logging +import os +import os.path as osp +from typing import Dict, List, Optional, Sequence, Tuple + +import numpy as np +import pycuda.autoinit # noqa: F401 +import pycuda.driver as cuda +import tensorrt as trt +import torch +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.config.schema import ComponentsConfig +from deployment.inference.gpu_resource_mixin import ( + GPUResourceMixin, + TensorRTResourceManager, + release_tensorrt_resources, +) +from deployment.primitives.artifacts import resolve_artifact_path +from deployment.primitives.device import DeviceSpec +from deployment.primitives.tensorrt_plugins import load_tensorrt_plugin_libraries +from deployment.projects.bevfusion.inference.bevfusion_inference_pipeline import BEVFusionDeploymentPipeline +from deployment.projects.bevfusion.inference.trt_profiling import ( + _SPARSE_BUCKET_ORDER, + _scale_dense_substages, + _sum_layers_by_stage, + _summarize_sparse_layers, + _TRTLayerProfiler, +) +from deployment.projects.bevfusion.io.component_utils import has_component, is_split_bevfusion_components +from deployment.projects.bevfusion.io.coors_contract import voxel_indices_xyz_to_graph_input_zyx + +logger = logging.getLogger(__name__) + + +def _env_truthy(key: str) -> bool: + return os.environ.get(key, "").strip().lower() in ("1", "true", "yes") + + +def _env_int(key: str, default: int) -> int: + try: + return int(os.environ.get(key, str(default)).strip()) + except ValueError: + return default + + +_TRT_DEBUG_SPLIT = _env_truthy("BEVFUSION_TRT_DEBUG_SPLIT") +_TRT_LOG_IO = _env_truthy("BEVFUSION_TRT_LOG_IO") +# Priority A — in-situ per-layer breakdown for the split sparse engine. +# BEVFUSION_TRT_SPARSE_PROFILE=1 attaches trt.IProfiler to the sparse context +# BEVFUSION_TRT_SPARSE_PROFILE_EVERY=1 log breakdown on every frame (default: every 10 frames) +# This env var is a sanity overlay when running the real eval path (step 5). +_TRT_SPARSE_PROFILE = _env_truthy("BEVFUSION_TRT_SPARSE_PROFILE") +_TRT_SPARSE_PROFILE_EVERY = max(1, _env_int("BEVFUSION_TRT_SPARSE_PROFILE_EVERY", 10)) +# First N eval frames: print pooled-voxel + lidar_bev stats to stdout (align with PyTorch pipeline). +_TRT_TENSOR_LOG_FRAMES = max(0, _env_int("BEVFUSION_TRT_TENSOR_LOG_FRAMES", 2)) +_TRT_TENSOR_LOG_PREFIX = "[BEVFUSION][TensorRT][tensors]" + + +def _np_tensor_stats(arr: np.ndarray, name: str) -> str: + """Compact numpy stats for debug lines (matches PyTorch _tensor_stats fields).""" + a = np.asarray(arr, dtype=np.float64).ravel() + nz = int(np.count_nonzero(a)) + return ( + f"{_TRT_TENSOR_LOG_PREFIX} {name}: shape={arr.shape} dtype={arr.dtype} " + f"min={float(a.min()):.4f} max={float(a.max()):.4f} " + f"mean={float(a.mean()):.4f} std={float(a.std()):.4f} " + f"abs_mean={float(np.mean(np.abs(a))):.4f} " + f"nonzero={nz}/{a.size}" + ) + + +def _list_trt_io_names(engine: trt.ICudaEngine) -> Tuple[List[str], List[str]]: + """Return (input_names, output_names) in TensorRT tensor index order.""" + inputs: List[str] = [] + outputs: List[str] = [] + for i in range(engine.num_io_tensors): + name = engine.get_tensor_name(i) + if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: + inputs.append(name) + else: + outputs.append(name) + return inputs, outputs + + +def _pick_bound_input_name(engine: trt.ICudaEngine, expected_in_order: Sequence[str]) -> str: + """Match deploy_cfg input names to the engine; avoid relying on arbitrary TRT ordering.""" + found, _out = _list_trt_io_names(engine) + for want in expected_in_order: + if want in found: + return want + if len(found) == 1: + if expected_in_order and found[0] != expected_in_order[0]: + logger.warning( + "TensorRT dense engine input is %r but deploy_cfg expects %r — using engine binding. " + "If mAP=0, verify ONNX export names match deploy components.bevfusion_dense.io.inputs.", + found[0], + expected_in_order[0], + ) + return found[0] + raise RuntimeError(f"Could not map deploy_cfg inputs {list(expected_in_order)} to engine inputs {found}") + + +def _log_engine_schema(tag: str, engine: trt.ICudaEngine) -> None: + ins, outs = _list_trt_io_names(engine) + lines = [f"[trt-io] {tag} inputs={ins} outputs={outs}"] + for name in ins + outs: + shp = engine.get_tensor_shape(name) + dt = engine.get_tensor_dtype(name) + lines.append(f"[trt-io] {name}: shape={shp} dtype={dt}") + logger.warning("\n".join(lines)) + + +def _log_engine_input_dtypes_line(tag: str, engine: trt.ICudaEngine) -> None: + """Log binding dtypes (P1: FP32 vs FP16 voxels for split sparse engines). + + HALF voxel bindings are supported via ``_host_buffer_for_engine_tensor``; this line makes + the contract visible without enabling ``BEVFUSION_TRT_LOG_IO=1``. + """ + parts: List[str] = [] + for i in range(engine.num_io_tensors): + name = engine.get_tensor_name(i) + if engine.get_tensor_mode(name) != trt.TensorIOMode.INPUT: + continue + dt = engine.get_tensor_dtype(name) + parts.append(f"{name}={dt}") + ln = name.lower() + if "voxel" in ln and "num" not in ln and dt == trt.DataType.HALF: + logger.warning( + "[trt-io] %s: voxel-like input %r is HALF — host numpy is cast before " + "``set_tensor_address`` (see INFO ``casting host buffer`` on first infer). " + "If that cast is missing, ImplicitGemm inputs corrupt (lidar_bev explodes).", + tag, + name, + ) + if parts: + logger.info("[trt-io] %s engine INPUT dtypes: %s", tag, ", ".join(parts)) + + +class BEVFusionTensorRTPipeline(GPUResourceMixin, BEVFusionDeploymentPipeline): + """TensorRT-based BEVFusion pipeline. + + Single engine (full graph) or split sparse + dense engines. + """ + + def __init__( + self, + pytorch_model: torch.nn.Module, + tensorrt_dir: str, + device: DeviceSpec, + components_cfg: ComponentsConfig, + plugin_libraries: Tuple[str, ...] = (), + ) -> None: + super().__init__(pytorch_model=pytorch_model, backend_type=Backend.TENSORRT, device=device) + + self.tensorrt_dir = tensorrt_dir + self._components_cfg = components_cfg + self._plugin_libraries = plugin_libraries + self._trt_logger = trt.Logger(trt.Logger.WARNING) + split_layout = is_split_bevfusion_components(components_cfg) + merged_engine_available = False + if split_layout and has_component(components_cfg, "bevfusion_main_body"): + merged_engine_path = resolve_artifact_path( + base_dir=tensorrt_dir, + components_cfg=components_cfg, + component_name="bevfusion_main_body", + file_key="engine_file", + ) + merged_engine_available = osp.exists(merged_engine_path) + self._split = split_layout and not merged_engine_available + self._engine = None + self._context = None + self._engine_sparse = None + self._context_sparse = None + self._engine_dense = None + self._context_dense = None + + self._start_event = cuda.Event() + self._end_event = cuda.Event() + # Split-engine GPU intervals (same stream as each TRT execute, excludes D2H). + self._sparse_ev_s = cuda.Event() + self._sparse_ev_e = cuda.Event() + self._dense_ev_s = cuda.Event() + self._dense_ev_e = cuda.Event() + self._last_split_sparse_gpu_ms: float = 0.0 + self._last_split_dense_gpu_ms: float = 0.0 + self._split_debug_frames_done: int = 0 + self._split_debug_max: int = max(0, _env_int("BEVFUSION_TRT_DEBUG_SPLIT_FRAMES", 2)) + self._split_tensor_log_frames_done: int = 0 + # Priority A: accumulators for sparse encoder bucket breakdown across eval frames. + self._sparse_profile_frame_count: int = 0 + self._sparse_profile_bucket_sum: Dict[str, float] = {b: 0.0 for b in _SPARSE_BUCKET_ORDER} + self._sparse_profile_top_layers: Dict[str, float] = {} # name -> accumulated ms + self._last_sparse_profile_buckets: Dict[str, float] = {} + + self._load_tensorrt_engine() + logger.info(f"BEVFusion TensorRT pipeline initialized from: {tensorrt_dir} (split={self._split})") + + def _load_tensorrt_engine(self) -> None: + load_tensorrt_plugin_libraries(logger, self._plugin_libraries) + trt.init_libnvinfer_plugins(self._trt_logger, "") + runtime = trt.Runtime(self._trt_logger) + + if self._split: + sparse_path = resolve_artifact_path( + base_dir=self.tensorrt_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_sparse", + file_key="engine_file", + ) + dense_path = resolve_artifact_path( + base_dir=self.tensorrt_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_dense", + file_key="engine_file", + ) + if not osp.exists(sparse_path): + raise FileNotFoundError(f"Sparse TensorRT engine not found: {sparse_path}") + if not osp.exists(dense_path): + raise FileNotFoundError(f"Dense TensorRT engine not found: {dense_path}") + + with open(sparse_path, "rb") as f: + self._engine_sparse = runtime.deserialize_cuda_engine(f.read()) + with open(dense_path, "rb") as f: + self._engine_dense = runtime.deserialize_cuda_engine(f.read()) + if self._engine_sparse is None or self._engine_dense is None: + raise RuntimeError("Failed to deserialize split TensorRT engines") + + self._context_sparse = self._engine_sparse.create_execution_context() + self._context_dense = self._engine_dense.create_execution_context() + if self._context_sparse is None or self._context_dense is None: + raise RuntimeError("Failed to create TensorRT contexts for split engines") + logger.info("Loaded split TensorRT engines: %s , %s", sparse_path, dense_path) + assert self._engine_sparse is not None and self._engine_dense is not None + _log_engine_input_dtypes_line("bevfusion_sparse", self._engine_sparse) + _log_engine_input_dtypes_line("bevfusion_dense", self._engine_dense) + if _TRT_LOG_IO: + _log_engine_schema("bevfusion_sparse", self._engine_sparse) + _log_engine_schema("bevfusion_dense", self._engine_dense) + return + + engine_path = resolve_artifact_path( + base_dir=self.tensorrt_dir, + components_cfg=self._components_cfg, + component_name="bevfusion_main_body", + file_key="engine_file", + ) + if not osp.exists(engine_path): + raise FileNotFoundError(f"TensorRT engine not found: {engine_path}") + + with open(engine_path, "rb") as f: + self._engine = runtime.deserialize_cuda_engine(f.read()) + if self._engine is None: + raise RuntimeError(f"Failed to deserialize engine: {engine_path}") + + self._context = self._engine.create_execution_context() + if self._context is None: + raise RuntimeError("Failed to create TensorRT execution context (OOM?)") + + logger.info(f"Loaded TensorRT engine: {engine_path}") + + @staticmethod + def _trt_dtype_to_numpy(trt_dtype: trt.DataType) -> np.dtype: + """Map TensorRT dtype to numpy dtype for correctly sized host buffers.""" + try: + return np.dtype(trt.nptype(trt_dtype)) + except Exception: + # Safe fallback for older/newer TRT dtype variations. + mapping = {} + for key, npdt in ( + ("FLOAT", np.float32), + ("HALF", np.float16), + ("INT8", np.int8), + ("INT32", np.int32), + ("BOOL", np.bool_), + ("UINT8", np.uint8), + ("FP8", np.float16), + ("BF16", np.float16), + ("INT64", np.int64), + ): + dt = getattr(trt.DataType, key, None) + if dt is not None: + mapping[dt] = npdt + return np.dtype(mapping.get(trt_dtype, np.float32)) + + def _host_buffer_for_engine_tensor(self, engine: trt.ICudaEngine, tensor_name: str, arr: np.ndarray) -> np.ndarray: + """Cast / layout host memory to match *engine* binding dtype (critical for FP16 engines). + + Split sparse ONNX is often traced with FP32 voxels, but TensorRT ``fp16`` builds may bind + ``voxels`` as ``HALF``. Feeding float32 nbytes into a HALF binding misaligns the GPU + buffer and corrupts the first ImplicitGemm inputs (lidar_bev explosion while numpy + voxel stats still look sane). + """ + trt_dtype = engine.get_tensor_dtype(tensor_name) + want = self._trt_dtype_to_numpy(trt_dtype) + if arr.dtype != want: + logger.info( + "[trt-io] casting host buffer for tensor %r: numpy %s → %s (engine binding %s)", + tensor_name, + arr.dtype, + want, + trt_dtype, + ) + arr = np.asarray(arr, dtype=want) + if not arr.flags["C_CONTIGUOUS"]: + arr = np.ascontiguousarray(arr) + return arr + + def _trt_infer_voxel_inputs( + self, + engine: trt.ICudaEngine, + context: trt.IExecutionContext, + voxels_np: np.ndarray, + coors_np: np.ndarray, + num_points_np: np.ndarray, + profiler: Optional[_TRTLayerProfiler], + gpu_interval_events: Optional[Tuple[cuda.Event, cuda.Event]], + ) -> Dict[str, np.ndarray]: + input_map: Dict[str, np.ndarray] = {} + output_names: List[str] = [] + for i in range(engine.num_io_tensors): + name = engine.get_tensor_name(i) + if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: + ln = name.lower() + if "voxel" in ln and "num" not in ln: + input_map[name] = voxels_np + elif "coor" in ln: + input_map[name] = coors_np + elif "num" in ln: + input_map[name] = num_points_np + else: + output_names.append(name) + return self._trt_infer_bound(engine, context, input_map, output_names, profiler, gpu_interval_events) + + def _trt_infer_named_input( + self, + engine: trt.ICudaEngine, + context: trt.IExecutionContext, + input_map: Dict[str, np.ndarray], + profiler: Optional[_TRTLayerProfiler], + gpu_interval_events: Optional[Tuple[cuda.Event, cuda.Event]], + ) -> Dict[str, np.ndarray]: + output_names: List[str] = [] + for i in range(engine.num_io_tensors): + name = engine.get_tensor_name(i) + if engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT: + output_names.append(name) + return self._trt_infer_bound(engine, context, input_map, output_names, profiler, gpu_interval_events) + + def _trt_infer_bound( + self, + engine: trt.ICudaEngine, + context: trt.IExecutionContext, + input_map: Dict[str, np.ndarray], + output_names: List[str], + profiler: Optional[_TRTLayerProfiler], + gpu_interval_events: Optional[Tuple[cuda.Event, cuda.Event]], + ) -> Dict[str, np.ndarray]: + input_map = {name: self._host_buffer_for_engine_tensor(engine, name, arr) for name, arr in input_map.items()} + for name, arr in input_map.items(): + context.set_input_shape(name, arr.shape) + + output_arrays: Dict[str, np.ndarray] = {} + for name in output_names: + shape = context.get_tensor_shape(name) + trt_dtype = engine.get_tensor_dtype(name) + np_dtype = self._trt_dtype_to_numpy(trt_dtype) + arr = np.empty(shape, dtype=np_dtype) + if not arr.flags["C_CONTIGUOUS"]: + arr = np.ascontiguousarray(arr) + output_arrays[name] = arr + + prev_profiler = None + if profiler is not None and hasattr(context, "profiler"): + prev_profiler = getattr(context, "profiler", None) + context.profiler = profiler + profiler.layer_times.clear() + + try: + with TensorRTResourceManager() as mgr: + d_inputs = {name: mgr.allocate(arr.nbytes) for name, arr in input_map.items()} + d_outputs = {name: mgr.allocate(arr.nbytes) for name, arr in output_arrays.items()} + stream = mgr.stream + + for name, arr in input_map.items(): + context.set_tensor_address(name, int(d_inputs[name])) + cuda.memcpy_htod_async(d_inputs[name], arr, stream) + + for name in output_names: + context.set_tensor_address(name, int(d_outputs[name])) + + if gpu_interval_events is not None: + gpu_interval_events[0].record(stream) + ok = context.execute_async_v3(stream_handle=stream.handle) + if not ok: + raise RuntimeError("TensorRT execute_async_v3 returned failure status.") + if gpu_interval_events is not None: + gpu_interval_events[1].record(stream) + + for name in output_names: + cuda.memcpy_dtoh_async(output_arrays[name], d_outputs[name], stream) + + mgr.synchronize() + finally: + # TensorRT rejects setProfiler(nullptr); only restore when a previous profiler existed. + if profiler is not None and hasattr(context, "profiler") and prev_profiler is not None: + context.profiler = prev_profiler + + return output_arrays + + @override + def run_bevfusion( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + profiler: _TRTLayerProfiler | None = None, + ) -> List[torch.Tensor]: + voxels_np = self.to_numpy(voxels, dtype=np.float32) + coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) + num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) + # Match ``extract_pts_feat``: mean-pool must not divide by zero (NaN BEV → dense NaN). + num_points_np = np.maximum(num_points_np, 1) + + if self._split: + assert self._engine_sparse is not None and self._context_sparse is not None + assert self._engine_dense is not None and self._context_dense is not None + + sparse_cfg = self._components_cfg.get_component("bevfusion_sparse") + dense_cfg = self._components_cfg.get_component("bevfusion_dense") + exp_sparse_out = [o.name for o in sparse_cfg.io.outputs] + exp_dense_in = [i.name for i in dense_cfg.io.inputs] + + do_tensor_log = _TRT_TENSOR_LOG_FRAMES > 0 and self._split_tensor_log_frames_done < _TRT_TENSOR_LOG_FRAMES + if do_tensor_log: + self._split_tensor_log_frames_done += 1 + fi = self._split_tensor_log_frames_done + print( + f"{_TRT_TENSOR_LOG_PREFIX} frame={fi}/{_TRT_TENSOR_LOG_FRAMES} " + f"(sparse TRT engine → lidar_bev → dense TRT engine → bbox/score/label)" + ) + voxelize_reduce = getattr(self.pytorch_model, "voxelize_reduce", True) + if voxelize_reduce and voxels_np.ndim == 3: + # Match pytorch.py: [N,P,C].sum(1) / npt with npt [N,1] → [N,C] (not [N] / [N,C]). + npt = np.maximum(num_points_np.astype(np.float32).reshape(-1, 1), 1.0) + voxel_feat_np = voxels_np.sum(axis=1, keepdims=False) / npt + print(_np_tensor_stats(voxel_feat_np, "voxel_features_input (numpy mean-pool, same as PyTorch)")) + elif voxels_np.ndim == 2: + print( + _np_tensor_stats( + voxels_np, + "voxel_features_input (already [N,C], no per-point dim — same as fed to TRT)", + ) + ) + else: + print( + f"{_TRT_TENSOR_LOG_PREFIX} voxel_features_input: skipped " + f"(voxelize_reduce={voxelize_reduce}, voxels_ndim={voxels_np.ndim})" + ) + + # Sparse (spconv) engine: CUDA-timed separately. If BEVFUSION_TRT_SPARSE_PROFILE=1, + # also attach a dedicated IProfiler here so we can answer Priority A's question + # ("where does the sparse time actually go?") without a separate run. + sparse_profiler: Optional[_TRTLayerProfiler] = _TRTLayerProfiler() if _TRT_SPARSE_PROFILE else None + sparse_out = self._trt_infer_voxel_inputs( + self._engine_sparse, + self._context_sparse, + voxels_np, + coors_np, + num_points_np, + profiler=sparse_profiler, + gpu_interval_events=(self._sparse_ev_s, self._sparse_ev_e), + ) + if sparse_profiler is not None: + self._record_sparse_profile(sparse_profiler.layer_times) + if len(sparse_out) != 1: + raise RuntimeError(f"Sparse engine: expected 1 output, got {list(sparse_out.keys())}") + bev_name = next(iter(sparse_out)) + if exp_sparse_out and bev_name not in exp_sparse_out: + logger.warning( + "[trt-split] sparse engine output tensor is %r but deploy_cfg bevfusion_sparse.io.outputs " + "names=%s — check ONNX export / TRT binding names.", + bev_name, + exp_sparse_out, + ) + bev_arr = np.ascontiguousarray(sparse_out[bev_name].astype(np.float32)) + + if do_tensor_log: + bn = bev_arr.reshape(-1) + print(_np_tensor_stats(bev_arr, f"sparse_encoder_output ({bev_name}, TRT sparse engine)")) + if bool(np.isnan(bn).any()) or bool(np.isinf(bn).any()): + print( + f"{_TRT_TENSOR_LOG_PREFIX} WARNING: lidar_bev has nan={bool(np.isnan(bn).any())} " + f"inf={bool(np.isinf(bn).any())}" + ) + + do_split_dbg = _TRT_DEBUG_SPLIT and self._split_debug_frames_done < self._split_debug_max + if do_split_dbg: + self._split_debug_frames_done += 1 + if not do_tensor_log: + bn = bev_arr.reshape(-1) + logger.warning( + "[BEVFUSION][TensorRT][debug-split] frame=%d/%d sparse->dense %s: shape=%s dtype=%s " + "min=%.6f max=%.6f mean=%.6f std=%.6f abs_mean=%.6f nan=%s inf=%s", + self._split_debug_frames_done, + self._split_debug_max, + bev_name, + bev_arr.shape, + bev_arr.dtype, + float(bn.min()), + float(bn.max()), + float(bn.mean()), + float(bn.std()), + float(np.mean(np.abs(bn))), + bool(np.isnan(bn).any()), + bool(np.isinf(bn).any()), + ) + + dense_in_name = _pick_bound_input_name(self._engine_dense, exp_dense_in) + + if do_split_dbg: + ctx = self._context_dense + exp_shape = tuple(ctx.get_tensor_shape(dense_in_name)) + logger.warning( + "[BEVFUSION][TensorRT][debug-split] dense input %r engine_expected_shape=%s feed_shape=%s " + "deploy_cfg_inputs=%s", + dense_in_name, + exp_shape, + bev_arr.shape, + exp_dense_in, + ) + if tuple(bev_arr.shape) != exp_shape and not any(d < 0 for d in exp_shape): + logger.warning( + "[BEVFUSION][TensorRT][debug-split] SHAPE MISMATCH: lidar_bev numpy shape %s vs TRT context %s — " + "dense engine will error or broadcast wrong; common cause: H×W vs export grid.", + bev_arr.shape, + exp_shape, + ) + + dense_out = self._trt_infer_named_input( + self._engine_dense, + self._context_dense, + {dense_in_name: bev_arr}, + profiler, + gpu_interval_events=(self._dense_ev_s, self._dense_ev_e), + ) + + self._sparse_ev_e.synchronize() + self._dense_ev_e.synchronize() + self._last_split_sparse_gpu_ms = float(self._sparse_ev_e.time_since(self._sparse_ev_s)) + self._last_split_dense_gpu_ms = float(self._dense_ev_e.time_since(self._dense_ev_s)) + + expected_output_names = [out.name for out in dense_cfg.io.outputs] + out_keys = list(dense_out.keys()) + ordered_names = [n for n in expected_output_names if n in dense_out] + ordered_names += [n for n in out_keys if n not in ordered_names] + tensors = [torch.from_numpy(dense_out[name]).to(self.torch_device) for name in ordered_names] + if do_tensor_log and tensors: + for i, name in enumerate(ordered_names): + t = tensors[i].detach() + t_f = t.float().reshape(-1) + extra = "" + if name == "bbox_pred" and t.ndim >= 2 and t.shape[0] >= 2: + cx = t[0].float().reshape(-1) + cy = t[1].float().reshape(-1) + extra = ( + f" center_x[min,max]=({float(cx.min()):.4f},{float(cx.max()):.4f}) " + f"center_y[min,max]=({float(cy.min()):.4f},{float(cy.max()):.4f})" + ) + if name == "label_pred": + lp = t.reshape(-1).long() + uniq = torch.unique(lp) + extra = ( + f" label_unique_count={int(uniq.numel())} label_min={int(lp.min())} " + f"label_max={int(lp.max())}" + ) + if name == "score": + extra = ( + f" score>0.1_count={int((t_f > 0.1).sum())} " f"score>0.5_count={int((t_f > 0.5).sum())}" + ) + print( + f"{_TRT_TENSOR_LOG_PREFIX} dense_out[{i}] {name} (TRT dense engine): " + f"shape={tuple(t.shape)} dtype={t.dtype} " + f"min={float(t_f.min().item()):.4f} max={float(t_f.max().item()):.4f} " + f"mean={float(t_f.mean().item()):.4f}{extra}" + ) + elif do_split_dbg and tensors: + for i, name in enumerate(ordered_names): + t = tensors[i].detach() + t_f = t.float().reshape(-1) + extra = "" + if name == "bbox_pred" and t.ndim >= 2 and t.shape[0] >= 2: + cx = t[0].float().reshape(-1) + cy = t[1].float().reshape(-1) + extra = f" center_x[min,max]=({float(cx.min())},{float(cx.max())}) center_y[min,max]=({float(cy.min())},{float(cy.max())})" + if name == "label_pred": + lp = t.reshape(-1).long() + uniq = torch.unique(lp) + extra = f" label_unique_count={int(uniq.numel())} label_min={int(lp.min())} label_max={int(lp.max())}" + if name == "score": + extra = f" score>0.1_count={int((t_f > 0.1).sum())} score>0.5_count={int((t_f > 0.5).sum())}" + logger.warning( + "[BEVFUSION][TensorRT][debug-split] dense_out[%s] %s: shape=%s dtype=%s min=%.6f max=%.6f mean=%.6f%s", + i, + name, + tuple(t.shape), + t.dtype, + float(t_f.min().item()), + float(t_f.max().item()), + float(t_f.mean().item()), + extra, + ) + return tensors + + engine = self._engine + context = self._context + assert engine is not None and context is not None + + output_arrays = self._trt_infer_voxel_inputs( + engine, + context, + voxels_np, + coors_np, + num_points_np, + profiler, + gpu_interval_events=(self._start_event, self._end_event), + ) + output_names = list(output_arrays.keys()) + + component_cfg = self._components_cfg.get_component("bevfusion_main_body") + expected_output_names = [out.name for out in component_cfg.io.outputs] + ordered_names = [n for n in expected_output_names if n in output_arrays] + ordered_names += [n for n in output_names if n not in ordered_names] + return [torch.from_numpy(output_arrays[name]).to(self.torch_device) for name in ordered_names] + + # Stage keys aligned with BEVFusionPyTorchPipeline for consistent Stage-wise Latency Breakdown. + # ``dense_engine_ms`` is the dense branch GPU time (split: CUDA events, merged: derived residual). + BEVFUSION_STAGE_KEYS = ( + "voxel_encoder_ms", + "sparse_encoder_ms", + "dense_engine_ms", + "backbone_ms", + "neck_ms", + "head_ms", + "post_scoring_ms", + "dense_unattributed_ms", + "bevfusion_ms", + ) + + @override + def run_model(self, preprocessed_input: Dict[str, torch.Tensor]) -> Tuple[List[torch.Tensor], Dict[str, float]]: + stage_latencies: Dict[str, float] = {k: 0.0 for k in self.BEVFUSION_STAGE_KEYS} + + profiler = _TRTLayerProfiler() + outputs = self.run_bevfusion( + preprocessed_input["voxels"], + preprocessed_input["coors"], + preprocessed_input["num_points_per_voxel"], + profiler=profiler, + ) + + # ------------------------------------------------------------------ + # Step 1: authoritative top-line GPU intervals (CUDA events). + # - bevfusion_ms : total TRT GPU time for the BEVFusion model. + # - sparse_encoder_ms / dense_engine_ms : the two top-level branches. + # Split has two physical engines (separate CUDA-event intervals). Merged + # is one engine, so we split its single interval by the per-layer profiler + # proportions (same classifier as the sub-stages below) — keeping every + # number on one clock and one naming contract. + # ------------------------------------------------------------------ + stage_sums = _sum_layers_by_stage(profiler.layer_times) if profiler.layer_times else None + + if self._split: + sparse_ms = self._last_split_sparse_gpu_ms + dense_ms = self._last_split_dense_gpu_ms + stage_latencies["bevfusion_ms"] = sparse_ms + dense_ms + else: + self._end_event.synchronize() + bevfusion_ms = float(self._end_event.time_since(self._start_event)) + stage_latencies["bevfusion_ms"] = bevfusion_ms + if stage_sums is not None: + total_raw = sum(stage_sums.values()) + sparse_frac = (stage_sums["sparse_encoder_ms"] / total_raw) if total_raw > 0.0 else 0.0 + sparse_ms = bevfusion_ms * sparse_frac + else: + sparse_ms = 0.0 + dense_ms = max(bevfusion_ms - sparse_ms, 0.0) + + stage_latencies["sparse_encoder_ms"] = sparse_ms + stage_latencies["dense_engine_ms"] = dense_ms + + # ------------------------------------------------------------------ + # Step 2: dense sub-stage breakdown — IDENTICAL path for merged & split. + # Per-layer (order-independent) classification gives the relative weight + # of backbone/neck/head/post_scoring, rescaled to the dense GPU interval. + # ------------------------------------------------------------------ + if stage_sums is not None: + dense_dist = _scale_dense_substages(stage_sums, dense_ms) + stage_latencies["backbone_ms"] = dense_dist["backbone_ms"] + stage_latencies["neck_ms"] = dense_dist["neck_ms"] + stage_latencies["head_ms"] = dense_dist["head_ms"] + stage_latencies["post_scoring_ms"] = dense_dist["post_scoring_ms"] + stage_latencies["dense_unattributed_ms"] = dense_dist["dense_unattributed_ms"] + else: + stage_latencies["dense_unattributed_ms"] = dense_ms + + # Align "Model" with the same interval semantics across merged/split TensorRT: + # report model_ms as the BEVFusion TRT GPU segment (not wall-clock Python overhead). + stage_latencies["model_ms"] = stage_latencies.get("bevfusion_ms", 0.0) + + return outputs, stage_latencies + + def _record_sparse_profile(self, layer_times: List[Tuple[str, float]]) -> None: + """Priority A in-situ overlay: accumulate sparse-engine bucket breakdown. + + We keep running sums across all eval frames so that after the run the user can + read off a 'mean sparse encoder bucket' right next to the normal latency table. + """ + if not layer_times: + return + buckets = _summarize_sparse_layers(layer_times) + self._last_sparse_profile_buckets = buckets + self._sparse_profile_frame_count += 1 + for b, ms in buckets.items(): + self._sparse_profile_bucket_sum[b] = self._sparse_profile_bucket_sum.get(b, 0.0) + ms + for name, ms in layer_times: + self._sparse_profile_top_layers[name] = self._sparse_profile_top_layers.get(name, 0.0) + ms + + if self._sparse_profile_frame_count % _TRT_SPARSE_PROFILE_EVERY == 0: + total = sum(buckets.values()) or 1e-9 + parts = [ + f"{b}={buckets[b]:.3f}ms ({buckets[b] / total * 100.0:.1f}%)" + for b in _SPARSE_BUCKET_ORDER + if buckets.get(b, 0.0) > 0.0 + ] + logger.info( + "[priority-a][sparse-profile] frame=%d sparse_layer_sum=%.3fms | %s", + self._sparse_profile_frame_count, + total, + " ".join(parts), + ) + + def print_sparse_profile_summary(self) -> None: + """Print Priority A mean-per-frame sparse-engine bucket breakdown. + + Called by the evaluator at the end of the run; no-op if the env var was off. + """ + n = self._sparse_profile_frame_count + if n <= 0: + return + logger.info("=" * 72) + logger.info("[priority-a] Sparse encoder in-situ bucket breakdown (mean/frame, n=%d)", n) + logger.info("=" * 72) + total_mean = sum(self._sparse_profile_bucket_sum.values()) / n + for b in _SPARSE_BUCKET_ORDER: + s = self._sparse_profile_bucket_sum.get(b, 0.0) + if s <= 0.0: + continue + mean = s / n + pct = (s / (total_mean * n)) * 100.0 if total_mean > 0.0 else 0.0 + logger.info(" %-20s %8.3f ms (%5.2f%%)", b, mean, pct) + logger.info(" %-20s %8.3f ms", "SUM", total_mean) + top_items = sorted(self._sparse_profile_top_layers.items(), key=lambda kv: -kv[1])[:10] + logger.info("Top 10 sparse layers (mean/frame):") + for name, acc in top_items: + logger.info(" %8.3f ms %s", acc / n, name) + logger.info("=" * 72) + + def _release_gpu_resources(self) -> None: + # Priority A — emit the sparse-profile summary before we tear engines down. + try: + self.print_sparse_profile_summary() + except Exception as exc: + logger.warning("[priority-a] sparse-profile summary failed: %s", exc) + for attr in ( + "_start_event", + "_end_event", + "_sparse_ev_s", + "_sparse_ev_e", + "_dense_ev_s", + "_dense_ev_e", + ): + if hasattr(self, attr): + try: + delattr(self, attr) + except Exception: + pass + if self._split: + release_tensorrt_resources( + engines={ + "sparse": self._engine_sparse, + "dense": self._engine_dense, + }, + contexts={ + "sparse": self._context_sparse, + "dense": self._context_dense, + }, + ) + else: + release_tensorrt_resources( + engines={"main": self._engine} if self._engine else None, + contexts={"main": self._context} if self._context else None, + ) diff --git a/deployment/projects/bevfusion/inference/trt_profiling.py b/deployment/projects/bevfusion/inference/trt_profiling.py new file mode 100644 index 000000000..3a678f01d --- /dev/null +++ b/deployment/projects/bevfusion/inference/trt_profiling.py @@ -0,0 +1,188 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Per-layer TensorRT profiling + BEVFusion stage attribution. + +Pure, order-independent helpers used by :class:`~...tensorrt_inference_pipeline.BEVFusionTensorRTPipeline` +to turn a TRT layer-time list into (a) Priority-A sparse-encoder buckets and (b) BEVFusion stage sums +(sparse / backbone / neck / head / post-scoring). Kept out of the inference pipeline so the runtime +path is not interleaved with ~200 lines of profiling/attribution logic. +""" + +import re +from typing import Dict, List, Tuple + +import tensorrt as trt + + +class _TRTLayerProfiler(trt.IProfiler): + """Collects per-layer execution times for TensorRT engine.""" + + def __init__(self) -> None: + try: + trt.IProfiler.__init__(self) + except Exception: + pass + self.layer_times: List[Tuple[str, float]] = [] + + def report_layer_time(self, layer_name: str, ms: float) -> None: + self.layer_times.append((str(layer_name), float(ms))) + + +# Priority A — bucket classification for sparse encoder in-situ profile. +_SPARSE_BUCKET_ORDER: Tuple[str, ...] = ( + "pair_gen", + "implicit_gemm_fp", + "scatter_nd", + "add", + "relu", + "cast", + "layout", + "other", +) + + +def _classify_sparse_bucket(layer_name: str) -> str: + """Match sparse encoder TRT layer names to a Priority A bucket. + + Keep patterns simple & case-insensitive — TRT forwards ONNX node names with + occasional prefixes, so pure substring matching is enough and cheap. + """ + n = layer_name.lower() + # Normalize common separators so ``ImplicitGemm`` / ``implicit-gemm`` / ``implicit gemm`` + # all collapse to ``implicitgemm`` before substring matching. + n_norm = n.replace("_", "").replace("-", "").replace(" ", "") + if "getindicepairsimplicitgemm" in n_norm or ("getindicepairs" in n_norm and "implicitgemm" not in n_norm): + return "pair_gen" + if "implicitgemm" in n_norm or "indiceconv" in n_norm: + return "implicit_gemm_fp" + if "scatternd" in n: + return "scatter_nd" + # Guard: "add"/"relu"/"cast" must be word-like to avoid matching paths. + if "relu" in n: + return "relu" + if "/add" in n or n.endswith("_add") or n.startswith("add"): + return "add" + if "/cast" in n or "_cast_" in n or n.startswith("cast"): + return "cast" + if any(k in n for k in ("reshape", "transpose", "concat", "slice", "gather", "squeeze", "unsqueeze")): + return "layout" + return "other" + + +def _summarize_sparse_layers(layer_times: List[Tuple[str, float]]) -> Dict[str, float]: + """Sum sparse TRT layer times per Priority A bucket (ms per frame).""" + sums: Dict[str, float] = {b: 0.0 for b in _SPARSE_BUCKET_ORDER} + for layer_name, ms in layer_times: + sums[_classify_sparse_bucket(layer_name)] += ms + return sums + + +# ============================================================================ +# Unified BEVFusion stage attribution (merged & split use the SAME logic). +# ---------------------------------------------------------------------------- +# Grounded in the BEVFusion ONNX module hierarchy, which is IDENTICAL for the +# merged full graph and the split dense graph (the merged graph only adds +# ``sparse/`` and ``dense/`` prefixes): +# pts_middle_encoder / spconv / ImplicitGemm ... -> sparse encoder +# pts_backbone (``blocks``) -> backbone +# pts_neck (``deblocks``) -> neck +# bbox_head (decoder / prediction_heads / heatmap_head) -> head +# score ops (sigmoid / one_hot / query_*) -> post scoring +# +# Classification is per-layer and ORDER-INDEPENDENT. This is the critical +# property: TensorRT freely fuses/reorders layers, so the previous order-based +# state machine mis-attributed cost (e.g. one early ``bbox_head`` layer flipped +# the whole stream to "head" and starved backbone/neck). A pure substring +# bucket per layer is stable regardless of fusion/order. +# ============================================================================ + +_BEVFUSION_DENSE_SUBSTAGE_KEYS: Tuple[str, ...] = ( + "backbone_ms", + "neck_ms", + "head_ms", + "post_scoring_ms", +) +_STAGE_OTHER = "other_ms" + + +def _classify_bevfusion_layer(layer_name: str) -> str: + """Classify one TensorRT layer into a BEVFusion stage key (order-independent). + + Returns one of: ``sparse_encoder_ms``, ``backbone_ms``, ``neck_ms``, + ``head_ms``, ``post_scoring_ms``, or ``other_ms`` (shape/glue, ~0 GPU time). + """ + n = layer_name.lower() + # Normalize separators so ``ImplicitGemm``/``implicit_gemm``/``getindicepairs`` all match. + nn = n.replace("_", "").replace("-", "").replace(" ", "") + + if any( + k in n + for k in ( + "pts_middle_encoder", + "middle_encoder", + "spconv", + "sparse_conv", + "subm", + "encoder_layer", + "conv_input", + "conv_out", + ) + ) or any(k in nn for k in ("implicitgemm", "getindicepairs", "scatternd")): + return "sparse_encoder_ms" + + # Neck before backbone: ``deblocks`` contains the substring ``blocks``. + if "pts_neck" in n or "deblocks" in n: + return "neck_ms" + + if "pts_backbone" in n or re.search(r"(^|[/.])blocks([/.]|$)", n): + return "backbone_ms" + + # ``bbox_head`` covers the transformer decoder, prediction_heads and heatmap_head. + if "bbox_head" in n: + return "head_ms" + + # Post-scoring ops typically live OUTSIDE bbox_head (top-level sigmoid/one_hot/query_*). + if any(k in nn for k in ("queryheatmapscore", "querylabels", "onehot")) or any( + k in n for k in ("sigmoid", "/topk", "argmax") + ): + return "post_scoring_ms" + + return _STAGE_OTHER + + +def _sum_layers_by_stage(layer_times: List[Tuple[str, float]]) -> Dict[str, float]: + """Sum profiler layer times into BEVFusion stage buckets (order-independent).""" + sums: Dict[str, float] = { + "sparse_encoder_ms": 0.0, + "backbone_ms": 0.0, + "neck_ms": 0.0, + "head_ms": 0.0, + "post_scoring_ms": 0.0, + _STAGE_OTHER: 0.0, + } + for layer_name, ms in layer_times: + sums[_classify_bevfusion_layer(layer_name)] += ms + return sums + + +def _scale_dense_substages(stage_sums: Dict[str, float], dense_total_ms: float) -> Dict[str, float]: + """Distribute the (CUDA-timed) dense total across backbone/neck/head/post_scoring. + + The per-layer profiler sums give the RELATIVE weight of each dense stage; we + rescale them so they add up exactly to ``dense_total_ms`` (the authoritative + GPU interval). ``other`` (shape/glue, ~0 GPU time) is absorbed proportionally, + so ``dense_unattributed_ms`` stays 0 whenever named stages are present. + """ + out: Dict[str, float] = {k: 0.0 for k in _BEVFUSION_DENSE_SUBSTAGE_KEYS} + out["dense_unattributed_ms"] = 0.0 + if dense_total_ms <= 0.0: + return out + + named_sum = sum(stage_sums.get(k, 0.0) for k in _BEVFUSION_DENSE_SUBSTAGE_KEYS) + if named_sum > 0.0: + scale = dense_total_ms / named_sum + for k in _BEVFUSION_DENSE_SUBSTAGE_KEYS: + out[k] = stage_sums.get(k, 0.0) * scale + out["dense_unattributed_ms"] = 0.0 + else: + out["dense_unattributed_ms"] = dense_total_ms + return out diff --git a/deployment/projects/bevfusion/io/__init__.py b/deployment/projects/bevfusion/io/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/projects/bevfusion/io/component_utils.py b/deployment/projects/bevfusion/io/component_utils.py new file mode 100644 index 000000000..1ee0f4467 --- /dev/null +++ b/deployment/projects/bevfusion/io/component_utils.py @@ -0,0 +1,100 @@ +"""Helpers for BEVFusion deploy ``components`` layout.""" + +from __future__ import annotations + +from typing import Any, Mapping + +from deployment.config.schema import ComponentsConfig + + +def is_split_bevfusion_components(components_cfg: ComponentsConfig) -> bool: + """True when deploy config uses sparse + dense ONNX/TRT (route 1), not a single main_body.""" + names = set(components_cfg.component_names()) + return "bevfusion_sparse" in names and "bevfusion_dense" in names + + +def should_merge_split_bevfusion(deploy_cfg: Mapping[str, Any]) -> bool: + """Return True when deploy config requests split->single export/eval merge.""" + merge_raw = deploy_cfg.get("bevfusion_merge", deploy_cfg.get("merge_bevfusion", deploy_cfg.get("merge", False))) + if isinstance(merge_raw, Mapping): + return bool(merge_raw.get("enabled", False)) + return bool(merge_raw) + + +def has_component(components_cfg: ComponentsConfig, component_name: str) -> bool: + """Return True if the component exists.""" + try: + components_cfg.get_component(component_name) + return True + except KeyError: + return False + + +def maybe_add_merged_main_body_component( + *, + deploy_cfg: Mapping[str, Any], + components_cfg: ComponentsConfig, +) -> ComponentsConfig: + """Optionally add merged main_body component while keeping split components. + + When ``bevfusion_merge`` is enabled and components are split, this function adds + ``bevfusion_main_body`` by reusing: + - split sparse input schema / TensorRT profile + - split dense output schema + """ + if not is_split_bevfusion_components(components_cfg): + return components_cfg + if not should_merge_split_bevfusion(deploy_cfg): + return components_cfg + if has_component(components_cfg, "bevfusion_main_body"): + return components_cfg + + sparse_cfg = components_cfg.get_component("bevfusion_sparse") + dense_cfg = components_cfg.get_component("bevfusion_dense") + + merge_raw = deploy_cfg.get("bevfusion_merge", deploy_cfg.get("merge_bevfusion", deploy_cfg.get("merge", {}))) + merge_cfg = merge_raw if isinstance(merge_raw, Mapping) else {} + onnx_file = str(merge_cfg.get("onnx_file", "bevfusion_lidar.onnx")) + engine_file = str(merge_cfg.get("engine_file", "bevfusion_lidar.engine")) + + merged_main_body = { + "bevfusion_main_body": { + "onnx_file": onnx_file, + "engine_file": engine_file, + "io": { + "inputs": [{"name": inp.name, "dtype": inp.dtype} for inp in sparse_cfg.io.inputs], + "outputs": [{"name": out.name, "dtype": out.dtype} for out in dense_cfg.io.outputs], + "dynamic_axes": dict(sparse_cfg.io.dynamic_axes), + }, + "tensorrt_profile": { + name: { + "min_shape": list(profile.min_shape), + "opt_shape": list(profile.opt_shape), + "max_shape": list(profile.max_shape), + } + for name, profile in sparse_cfg.tensorrt_profile.items() + }, + } + } + + raw_components = {} + for name, comp in components_cfg.items(): + raw_components[name] = { + "onnx_file": comp.onnx_file, + "engine_file": comp.engine_file, + "io": { + "inputs": [{"name": inp.name, "dtype": inp.dtype} for inp in comp.io.inputs], + "outputs": [{"name": out.name, "dtype": out.dtype} for out in comp.io.outputs], + "dynamic_axes": dict(comp.io.dynamic_axes), + }, + "tensorrt_profile": { + k: { + "min_shape": list(v.min_shape), + "opt_shape": list(v.opt_shape), + "max_shape": list(v.max_shape), + } + for k, v in comp.tensorrt_profile.items() + }, + } + raw_components.update(merged_main_body) + return ComponentsConfig.from_dict(raw_components) diff --git a/deployment/projects/bevfusion/io/coors_contract.py b/deployment/projects/bevfusion/io/coors_contract.py new file mode 100644 index 000000000..7ad2ace90 --- /dev/null +++ b/deployment/projects/bevfusion/io/coors_contract.py @@ -0,0 +1,29 @@ +"""BEVFusion sparse ``coors`` layout for deploy / ONNX / TensorRT. + +Voxelization (``pts_voxel_layer``) returns indices as ``[x, y, z]`` (see +``dynamic_voxelize_kernel`` in ``bevfusion/ops/voxel``). + +Legacy Autoware-compatible ONNX expects graph **inputs** as ``[z, y, x]`` (no batch). +Inside the exported wrapper, indices are flipped to ``[x, y, z]`` and a batch column +is prepended before ``pts_middle_encoder`` (``sparse_shape`` is ``[H, W, D]``). + +PyTorch evaluation uses ``[batch, x, y, z]`` directly and does not use this module. +""" + +from __future__ import annotations + +import torch + + +def voxel_indices_xyz_to_graph_input_zyx(coors: torch.Tensor) -> torch.Tensor: + """``[M, 3]`` voxel indices ``[x, y, z]`` → graph input ``[z, y, x]``.""" + if coors.ndim != 2 or coors.shape[1] != 3: + return coors + return coors.flip(dims=[-1]).contiguous() + + +def graph_input_zyx_to_model_indices_xyz(coors: torch.Tensor) -> torch.Tensor: + """``[M, 3]`` graph input ``[z, y, x]`` → model indices ``[x, y, z]`` (wrapper flip).""" + if coors.ndim != 2 or coors.shape[1] != 3: + return coors + return coors.flip(dims=[-1]).contiguous() diff --git a/deployment/projects/bevfusion/io/data_loader.py b/deployment/projects/bevfusion/io/data_loader.py new file mode 100644 index 000000000..5139fb992 --- /dev/null +++ b/deployment/projects/bevfusion/io/data_loader.py @@ -0,0 +1,82 @@ +"""BEVFusion DataLoader for deployment. + +Wraps MMDet3D Dataset to load point cloud data for BEVFusion inference. +Pipeline runs once per sample in load_sample(), avoiding redundant computation. +""" + +from __future__ import annotations + +import copy +from typing import Dict, List, Optional, Union + +import torch +from mmengine.config import Config +from mmengine.registry import DATASETS, init_default_scope +from typing_extensions import override + +from deployment.io.base_data_loader import BaseDataLoader + + +class BEVFusionDataLoader(BaseDataLoader): + """Deployment dataloader for BEVFusion using MMDet3D Dataset. + + Wraps the same Dataset used by training/testing, ensuring identical + GT and pipeline processing. + """ + + def __init__(self, info_file: str, model_cfg: Config) -> None: + super().__init__() + self.model_cfg = model_cfg + self.info_file = info_file + self.dataset = self._build_dataset(model_cfg, info_file) + + def _build_dataset(self, model_cfg: Config, info_file: str) -> torch.utils.data.Dataset: + init_default_scope("mmdet3d") + if not hasattr(model_cfg, "test_dataloader"): + raise ValueError("model_cfg must have 'test_dataloader' with dataset config") + dataset_cfg = copy.deepcopy(model_cfg.test_dataloader.dataset) + dataset_cfg["ann_file"] = info_file + dataset_cfg["test_mode"] = True + return DATASETS.build(dataset_cfg) + + @override + def load_sample(self, index: int) -> Dict[str, Union[torch.Tensor, Dict[str, object]]]: + if index >= len(self.dataset): + raise IndexError(f"Sample index {index} out of range (0-{len(self.dataset)-1})") + + data = self.dataset[index] + pipeline_inputs = data["inputs"] + points_tensor = pipeline_inputs["points"].to("cpu") + + data_samples = data["data_samples"] + metainfo = getattr(data_samples, "metainfo", None) + eval_ann_info = getattr(data_samples, "eval_ann_info", None) + ground_truth = dict(eval_ann_info) if eval_ann_info else {} + + return { + "points": points_tensor, + "metainfo": dict(metainfo) if metainfo else {}, + "ground_truth": ground_truth, + } + + @override + def preprocess( + self, sample: Dict[str, Union[torch.Tensor, Dict[str, object]]] + ) -> Dict[str, Union[torch.Tensor, Dict[str, object]]]: + return { + "points": sample["points"], + "metainfo": sample["metainfo"], + } + + @property + @override + def num_samples(self) -> int: + return len(self.dataset) + + @property + def class_names(self) -> List[str]: + if hasattr(self.dataset, "metainfo") and "classes" in self.dataset.metainfo: + return list(self.dataset.metainfo["classes"]) + if hasattr(self.model_cfg, "class_names"): + return list(self.model_cfg.class_names) + raise ValueError("class_names not found in dataset.metainfo or model_cfg") diff --git a/deployment/projects/bevfusion/io/model_loader.py b/deployment/projects/bevfusion/io/model_loader.py new file mode 100644 index 000000000..b995009bf --- /dev/null +++ b/deployment/projects/bevfusion/io/model_loader.py @@ -0,0 +1,65 @@ +"""BEVFusion model loading utilities for deployment.""" + +from __future__ import annotations + +import copy +import logging + +import torch +from mmengine.config import Config +from mmengine.registry import MODELS, init_default_scope +from mmengine.runner import load_checkpoint + +from deployment.primitives.device import DeviceSpec + +logger = logging.getLogger(__name__) + + +def _register_bevfusion_modules() -> None: + """Register BEVFusion and SparseConvolution modules into MMDet3D registries.""" + import projects.BEVFusion.bevfusion # noqa: F401 + import projects.SparseConvolution # noqa: F401 + + +def build_bevfusion_model( + model_cfg: Config, + checkpoint_path: str, + device: DeviceSpec, + *, + fuse_spconv_bn: bool = False, +) -> torch.nn.Module: + """Build a BEVFusion model from config and load checkpoint weights. + + Args: + model_cfg: MMEngine model configuration. + checkpoint_path: Path to .pth checkpoint file. + device: Target device. + fuse_spconv_bn: If True, fuse each ``SparseConvolution`` + ``BatchNorm1d`` pair in + ``pts_middle_encoder`` after ``load_checkpoint`` (eval-mode Conv-BN fold, a graph + optimization for the sparse ONNX export). + + Returns: + Loaded and eval-mode BEVFusion model. + """ + init_default_scope("mmdet3d") + _register_bevfusion_modules() + + model_config = copy.deepcopy(model_cfg.model) + model = MODELS.build(model_config) + + torch_device = device.to_torch_device() + model.to(torch_device) + + load_checkpoint(model, checkpoint_path, map_location=torch_device) + + if fuse_spconv_bn: + from deployment.projects.bevfusion.export.spconv_bn_fusion import fuse_spconv_bn_in_encoder + + encoder = getattr(model, "pts_middle_encoder", None) + if encoder is not None: + count = fuse_spconv_bn_in_encoder(encoder) + logger.info("Fused %d SparseConv-BN pair(s) in pts_middle_encoder", count) + + model.eval() + model.cfg = model_cfg + return model diff --git a/deployment/projects/bevfusion/runner.py b/deployment/projects/bevfusion/runner.py new file mode 100644 index 000000000..05a8ca6c0 --- /dev/null +++ b/deployment/projects/bevfusion/runner.py @@ -0,0 +1,92 @@ +"""BEVFusion-specific deployment runner.""" + +from __future__ import annotations + +import logging +from typing import Optional, Tuple + +import torch +from mmengine.config import Config + +from deployment.config.base import BaseDeploymentConfig +from deployment.evaluation.backend_executor import BackendExecutor +from deployment.export.contexts import ExportContext +from deployment.io.base_data_loader import BaseDataLoader +from deployment.projects.bevfusion.evaluation.evaluator import BEVFusionEvaluator +from deployment.projects.bevfusion.export.onnx_export_pipeline import BEVFusionONNXExportPipeline +from deployment.projects.bevfusion.export.tensorrt_export_pipeline import BEVFusionTensorRTExportPipeline +from deployment.projects.bevfusion.io.model_loader import build_bevfusion_model +from deployment.runtime.runner import BaseDeploymentRunner + +logger = logging.getLogger(__name__) + + +class BEVFusionDeploymentRunner(BaseDeploymentRunner): + """BEVFusion deployment runner. + + Constructs BEVFusion's model-specific ONNX/TensorRT export pipelines and injects them + into the project-agnostic ``BaseDeploymentRunner`` via its ``onnx_pipeline`` / + ``tensorrt_pipeline`` override hooks (BEVFusion needs wrapper modules, TopK constant + folding, coordinate flips, and split→merge ONNX composition that the generic whole-model + export cannot express). + + BEVFusion-only deploy-config flags (``fuse_spconv_bn``, ``spconv_do_sort``, + ``spconv_fuse_implicit_gemm_relu``) are read from the raw ``deploy_cfg`` passed in by the + entrypoint, since ``BaseDeploymentConfig`` only surfaces typed sections. + """ + + def __init__( + self, + data_loader: BaseDataLoader, + evaluator: BEVFusionEvaluator, + executor: BackendExecutor, + config: BaseDeploymentConfig, + model_cfg: Config, + deploy_cfg: Config, + module: str = "main_body", + plugin_libraries: Tuple[str, ...] = (), + onnx_pipeline: Optional[BEVFusionONNXExportPipeline] = None, + tensorrt_pipeline: Optional[BEVFusionTensorRTExportPipeline] = None, + ) -> None: + self._module = module + self._deploy_cfg = deploy_cfg + + # Construct the model-specific pipelines BEFORE super().__init__, because the base + # runner forwards them straight to the ExportOrchestrator (there is no post-init slot). + if onnx_pipeline is None: + onnx_pipeline = BEVFusionONNXExportPipeline(module=module) + if tensorrt_pipeline is None: + tensorrt_pipeline = BEVFusionTensorRTExportPipeline( + components_cfg=config.components_cfg, + plugin_libraries=tuple(plugin_libraries), + ) + + super().__init__( + data_loader=data_loader, + evaluator=evaluator, + executor=executor, + config=config, + model_cfg=model_cfg, + onnx_pipeline=onnx_pipeline, + tensorrt_pipeline=tensorrt_pipeline, + ) + + def load_pytorch_model(self, checkpoint_path: str, context: ExportContext) -> torch.nn.Module: + """Load the BEVFusion model onto the CUDA device for export. + + The base runner forwards the returned model to ``executor.set_pytorch_model`` after + export, so PyTorch/ONNX/TensorRT evaluation all reuse this reference. + """ + cuda_device = self.config.device_config.cuda + if cuda_device is None: + raise RuntimeError( + "BEVFusion requires a CUDA device for sparse convolution. Set devices.cuda in deploy config." + ) + + fuse_spconv_bn = bool(self._deploy_cfg.get("fuse_spconv_bn", False)) + return build_bevfusion_model( + model_cfg=self.model_cfg, + checkpoint_path=checkpoint_path, + device=cuda_device, + fuse_spconv_bn=fuse_spconv_bn, + ) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py b/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py new file mode 100644 index 000000000..bed1f4fd6 --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py @@ -0,0 +1,166 @@ +""" +CenterPoint FP16 Deployment Configuration - ConvNeXt Small Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/centerpoint-convnext/epoch_5_downsample_conv_first.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir="work_dirs/centerpoint-convnext/small/fp16-downsample-conv-first", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# +# ConvNeXt Small uses BackwardPillarFeatureNet with 10 input channels: +# base (5) + cluster_center (3) + voxel_center (2) = 10 +# Grid size: [1216, 1216, 1] +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 10], + opt_shape=[20000, 32, 10], + max_shape=[64000, 32, 10], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1216, 1216], + opt_shape=[1, 32, 1216, 1216], + max_shape=[1, 32, 1216, 1216], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=20, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=100, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=True, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py b/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py new file mode 100644 index 000000000..811e137c8 --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - ConvNeXt Standard Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/centerpoint-convnext/standard/epoch_30_standard.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="onnx", + work_dir="work_dirs/centerpoint-convnext/standard", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[64000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=100, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py b/deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py new file mode 100644 index 000000000..1fa465fcc --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - ResNet34 Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/centerpoint_resnet34_exp3.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="none", + work_dir="work_dirs/centerpoint_fp16_resnet_deployment_exp3", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[64000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=100, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py b/deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py new file mode 100644 index 000000000..162dc42d8 --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - ResNet34 Backbone (Base) +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "models/2_5/base/centerpoint_resnet34_base_2_5_epoch49.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir="work_dirs/centerpoint_fp16_resnet_deployment_base", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 10], + opt_shape=[20000, 32, 10], + max_shape=[96000, 32, 10], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=-1, + verbose=True, + backends=dict( + pytorch=dict( + enabled=False, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second.py new file mode 100644 index 000000000..c59e7bcc9 --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_second.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - SECOND Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/centerpoint_2_5.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="none", + work_dir="work_dirs/centerpoint_fp16_second_deployment_2_5", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[64000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=100, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py new file mode 100644 index 000000000..995b30b2d --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - SECOND Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "models/2_5/experiment_j6_gen2/second/epoch_30.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir="models/2_5/experiment_j6_gen2/second/fp16-deployment", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[64000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=100, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py new file mode 100644 index 000000000..a0d577a3f --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py @@ -0,0 +1,160 @@ +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "vivid/bench_comparison/centerpoint_2_6/epoch_29.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# Single literal for deployment output root (used before `export` exists). +_DEPLOY_WORK_DIR = "work_dirs/centerpoint_2_6_fp16" +_WORK_DIR = _DEPLOY_WORK_DIR.rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir=_DEPLOY_WORK_DIR, + onnx_path=_ONNX_DIR, + sample_idx=1, +) + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[96000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=17, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=500, + num_warmup=2, + verbose=True, + backends=dict( + pytorch=dict( + enabled=False, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py new file mode 100644 index 000000000..3e9475dfd --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - SECOND Backbone (Base) +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "models/2_5/base/centerpoint_second_base_2_5_epoch49.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir="work_dirs/centerpoint_fp16_second_deployment_2_5_base", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 10], + opt_shape=[20000, 32, 10], + max_shape=[96000, 32, 10], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=100, + verbose=True, + backends=dict( + pytorch=dict( + enabled=False, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py b/deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py new file mode 100644 index 000000000..149c30151 --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - SECOND Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "models/2_5/experiment_j6_gen2/vov57-v2-downsample/epoch_30.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir="work_dirs/centerpoint-vov57-v2-downsample/fp16-deployment", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[96000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=-1, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py b/deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py new file mode 100644 index 000000000..6f4dceace --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP16 Deployment Configuration - SECOND Backbone +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "models/2_5/experiment_j6_gen2/vov_epoch_30.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="both", + work_dir="work_dirs/centerpoint-vov99/fp16-deployment", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[96000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 1020, 1020], + opt_shape=[1, 32, 1020, 1020], + max_shape=[1, 32, 1020, 1020], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=-1, + verbose=True, + backends=dict( + pytorch=dict( + enabled=True, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp32.py b/deployment/projects/centerpoint/config/deploy_config_fp32.py new file mode 100644 index 000000000..1df466c44 --- /dev/null +++ b/deployment/projects/centerpoint/config/deploy_config_fp32.py @@ -0,0 +1,162 @@ +""" +CenterPoint FP32 Deployment Configuration +""" + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "vivid_model/best_checkpoint.pth" + +deploy_log_path = "deployment.log" + +# ============================================================================ +# Device settings +# ============================================================================ +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +# ============================================================================ +# Export Configuration +# ============================================================================ +export = dict( + mode="none", + work_dir="work_dirs/centerpoint_deployment_fp32", + onnx_path=None, + sample_idx=1, +) + +# Derived artifact directories +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Unified Component Configuration +# ============================================================================ +components = dict( + pts_voxel_encoder=dict( + onnx_file="pts_voxel_encoder.onnx", + engine_file="pts_voxel_encoder.engine", + io=dict( + inputs=[ + dict(name="input_features", dtype="float32"), + ], + outputs=[ + dict(name="pillar_features", dtype="float32"), + ], + dynamic_axes={ + "input_features": {0: "num_voxels", 1: "num_max_points"}, + "pillar_features": {0: "num_voxels"}, + }, + ), + tensorrt_profile=dict( + input_features=dict( + min_shape=[1000, 32, 11], + opt_shape=[20000, 32, 11], + max_shape=[64000, 32, 11], + ), + ), + ), + pts_backbone_neck_head=dict( + onnx_file="pts_backbone_neck_head.onnx", + engine_file="pts_backbone_neck_head.engine", + io=dict( + inputs=[ + dict(name="spatial_features", dtype="float32"), + ], + outputs=[ + dict(name="heatmap", dtype="float32"), + dict(name="reg", dtype="float32"), + dict(name="height", dtype="float32"), + dict(name="dim", dtype="float32"), + dict(name="rot", dtype="float32"), + dict(name="vel", dtype="float32"), + ], + dynamic_axes={ + "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, + "heatmap": {0: "batch_size", 2: "height", 3: "width"}, + "reg": {0: "batch_size", 2: "height", 3: "width"}, + "height": {0: "batch_size", 2: "height", 3: "width"}, + "dim": {0: "batch_size", 2: "height", 3: "width"}, + "rot": {0: "batch_size", 2: "height", 3: "width"}, + "vel": {0: "batch_size", 2: "height", 3: "width"}, + }, + ), + tensorrt_profile=dict( + spatial_features=dict( + min_shape=[1, 32, 760, 760], + opt_shape=[1, 32, 760, 760], + max_shape=[1, 32, 760, 760], + ), + ), + ), +) + +# ============================================================================ +# ONNX Export Settings +# ============================================================================ +onnx_config = dict( + opset_version=16, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# ============================================================================ +# TensorRT Build Settings +# ============================================================================ +tensorrt_config = dict( + precision_policy="fp32_tf32", + max_workspace_size=4 << 30, +) + +# ============================================================================ +# Evaluation Configuration +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=-1, + verbose=True, + backends=dict( + pytorch=dict( + enabled=False, + device=devices["cuda"], + ), + onnx=dict( + enabled=False, + device=devices["cuda"], + model_dir=_ONNX_DIR, + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# Verification Configuration +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1e-1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + onnx=[ + dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), + ], + trt=[ + dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), + ], + none=[], + ), +) diff --git a/deployment/projects/centerpoint/evaluation/evaluator.py b/deployment/projects/centerpoint/evaluation/evaluator.py index e3907b8cb..3920dda5e 100644 --- a/deployment/projects/centerpoint/evaluation/evaluator.py +++ b/deployment/projects/centerpoint/evaluation/evaluator.py @@ -1,185 +1,26 @@ -""" -CenterPoint Evaluator for deployment. +"""CenterPoint evaluator for deployment. + +Thin subclass of ``Detection3DEvaluator``: only ``print_results`` (the flat latency-breakdown +layout) is CenterPoint-specific; the metrics hooks (parse/accumulate/build/summarize) are shared +with the base (see ``deployment.evaluation.detection3d_evaluator``). """ import logging -from typing import Dict, List, Mapping -import numpy as np -from mmengine.config import Config from typing_extensions import override -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.evaluation.base_evaluator import ( - BaseEvaluator, - EvalResultDict, -) -from deployment.metrics.detection_3d_metrics import ( - Detection3DMetricsConfig, - Detection3DMetricsInterface, -) +from deployment.evaluation.base_evaluator import EvalResultDict +from deployment.evaluation.detection3d_evaluator import Detection3DEvaluator logger = logging.getLogger(__name__) -class CenterPointEvaluator(BaseEvaluator): - """Evaluator implementation for CenterPoint 3D detection. - - Uses the configured `Detection3DMetricsInterface` to compute metrics from pipeline outputs. - - Args: - model_cfg: Model configuration with class_names - metrics_config: Configuration for 3D detection metrics - executor: Backend execution primitives (a `CenterPointExecutor`), shared with - the verification runner. - """ - - def __init__( - self, - model_cfg: Config, - metrics_config: Detection3DMetricsConfig, - executor: BackendExecutor, - ) -> None: - """Initialize CenterPoint evaluator with model config, metrics config, and executor. - - Args: - model_cfg: Model configuration; must have class_names. - metrics_config: Configuration for 3D detection metrics (e.g. T4MetricV2). - executor: Backend execution primitives shared with the verification runner. - - Raises: - ValueError: If model_cfg does not have class_names. - """ - if not hasattr(model_cfg, "class_names"): - raise ValueError("class_names must be provided via model_cfg.class_names.") - - metrics_interface = Detection3DMetricsInterface(metrics_config) - - super().__init__( - metrics_interface=metrics_interface, - model_cfg=model_cfg, - executor=executor, - ) - - @override - def _parse_predictions(self, pipeline_output: object) -> List[Dict]: - """Return pipeline output as a list of prediction dicts (or empty list if not a list). - - Args: - pipeline_output: Raw output from the inference pipeline. - - Returns: - List of prediction dicts, or empty list if pipeline_output is not a list. - """ - return pipeline_output if isinstance(pipeline_output, list) else [] - - @override - def _parse_ground_truths(self, gt_data: Mapping[str, object]) -> List[Dict]: - """Convert gt_bboxes_3d and gt_labels_3d into list of dicts with bbox_3d and label. - - Args: - gt_data: Dict with 'gt_bboxes_3d' and 'gt_labels_3d'. - - Returns: - List of {"bbox_3d": [...], "label": int}. - - Raises: - KeyError: If gt_bboxes_3d or gt_labels_3d is missing. - """ - if "gt_bboxes_3d" not in gt_data: - raise KeyError("gt_bboxes_3d not found in ground truth data.") - if "gt_labels_3d" not in gt_data: - raise KeyError("gt_labels_3d not found in ground truth data.") - - gt_bboxes_3d = gt_data["gt_bboxes_3d"] - gt_labels_3d = gt_data["gt_labels_3d"] - - gt_bboxes_3d = np.asarray(gt_bboxes_3d, dtype=np.float32).reshape( - -1, np.asarray(gt_bboxes_3d).shape[-1] if np.asarray(gt_bboxes_3d).ndim > 1 else 7 - ) - gt_labels_3d = np.asarray(gt_labels_3d, dtype=np.int64).reshape(-1) - - ground_truths = [ - {"bbox_3d": gt_bboxes_3d[i].tolist(), "label": int(gt_labels_3d[i])} for i in range(len(gt_bboxes_3d)) - ] - return ground_truths - - @override - def _add_to_interface(self, predictions: List[Dict], ground_truths: List[Dict]) -> None: - """Add one frame of predictions and ground truths to the metrics interface. - - Args: - predictions: List of prediction dicts (bbox_3d, score, label). - ground_truths: List of ground truth dicts (bbox_3d, label). - """ - self.metrics_interface.add_frame(predictions, ground_truths) - - @override - def _build_results( - self, - latencies: List[float], - latency_breakdowns: List[Dict[str, float]], - num_samples: int, - ) -> EvalResultDict: - """Build evaluation result dict with mAP/mAPH, per-class AP, latency, and optional breakdown. - - Args: - latencies: Per-sample inference latencies (ms). - latency_breakdowns: Per-sample stage-wise latencies (optional). - num_samples: Number of evaluated samples. - - Returns: - EvalResultDict with mAP_by_mode, mAPH_by_mode, per_class_ap_by_mode, - detailed_metrics, latency stats, num_samples, and optionally latency_breakdown. - - Raises: - KeyError: If metrics summary is missing required keys. - """ - latency_stats = self.compute_latency_stats(latencies) - - map_results = self.metrics_interface.compute_metrics() - summary = self.metrics_interface.summary - summary_dict = summary.to_dict() - required_summary_keys = ("mAP_by_mode", "mAPH_by_mode", "per_class_ap_by_mode") - missing = [k for k in required_summary_keys if k not in summary_dict] - if missing: - raise KeyError(f"Missing required metrics summary keys: {missing}") - - result: EvalResultDict = { - "mAP_by_mode": summary_dict["mAP_by_mode"], - "mAPH_by_mode": summary_dict["mAPH_by_mode"], - "per_class_ap_by_mode": summary_dict["per_class_ap_by_mode"], - "detailed_metrics": map_results, - "latency": latency_stats, - "num_samples": num_samples, - } - - if latency_breakdowns: - result["latency_breakdown"] = self._compute_latency_breakdown(latency_breakdowns) - - return result - - @override - def summarize_for_comparison(self, results: EvalResultDict) -> List[str]: - """Summarize mAP/mAPH per mode for the cross-backend comparison.""" - lines: List[str] = [] - for mode, map_value in (results.get("mAP_by_mode") or {}).items(): - lines.append(f" mAP ({mode}): {map_value:.4f}") - for mode, maph_value in (results.get("mAPH_by_mode") or {}).items(): - lines.append(f" mAPH ({mode}): {maph_value:.4f}") - lines.extend(super().summarize_for_comparison(results)) - return lines +class CenterPointEvaluator(Detection3DEvaluator): + """Evaluator for CenterPoint 3D detection deployment.""" @override def print_results(self, results: EvalResultDict) -> None: - """Log evaluation results including metrics, latency, and breakdown. - - Args: - results: EvalResultDict from _build_results (mAP, latency, num_samples, etc.). - - Raises: - ValueError: If metrics report or latency is missing from results. - """ + """Log the metrics report, latency statistics, and stage-wise breakdown.""" metrics_report = self.metrics_interface.format_metrics_report() for line in metrics_report.rstrip().split("\n"): logger.info(line) @@ -188,26 +29,16 @@ def print_results(self, results: EvalResultDict) -> None: raise ValueError( "Latency statistics not found in results. Ensure that evaluation has been run with latency tracking." ) - latency_stats = results["latency"] - latency_dict = latency_stats.to_dict() - logger.info("") - logger.info("Latency Statistics:") - logger.info(" Mean: %.2f ms", latency_dict["mean_ms"]) - logger.info(" Std: %.2f ms", latency_dict["std_ms"]) - logger.info(" Min: %.2f ms", latency_dict["min_ms"]) - logger.info(" Max: %.2f ms", latency_dict["max_ms"]) - logger.info(" Median: %.2f ms", latency_dict["median_ms"]) + self._log_latency_stats(results) if "latency_breakdown" in results: breakdown_dict = results["latency_breakdown"].to_dict() - if breakdown_dict: logger.info("") logger.info("Stage-wise Latency Breakdown:") top_level_stages = {"preprocessing_ms", "model_ms", "postprocessing_ms"} for stage, stats_dict in breakdown_dict.items(): stage_name = stage.replace("_ms", "").replace("_", " ").title() - output_format = ( " %-18s: %.2f ± %.2f ms" if stage in top_level_stages else " %-16s: %.2f ± %.2f ms" ) diff --git a/deployment/projects/centerpoint/evaluation/executor.py b/deployment/projects/centerpoint/evaluation/executor.py index 092f0ef7d..fecade771 100644 --- a/deployment/projects/centerpoint/evaluation/executor.py +++ b/deployment/projects/centerpoint/evaluation/executor.py @@ -1,113 +1,29 @@ -""" -CenterPoint backend executor. +"""CenterPoint backend executor. -Implements the task-specific backend execution primitives (pipeline creation and -input preparation) for CenterPoint, shared by the evaluator and the verification -runner via `~deployment.evaluation.backend_executor.BackendExecutor`. +Thin subclass of ``PointDetectionExecutor``: declares the CenterPoint pipeline classes and +the head output-name lookup. Pipeline creation and ``(points, metainfo)`` input prep are shared +with the base (see ``deployment.evaluation.point_detection_executor``). """ -import logging -from typing import List, Mapping, Optional +from typing import List, Optional from typing_extensions import override -from deployment.config.enums import Backend -from deployment.config.schema import ComponentsConfig -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.evaluation.evaluator_types import InferenceInput, ModelSpec -from deployment.inference.base_inference_pipeline import BaseInferencePipeline -from deployment.io.base_data_loader import BaseDataLoader -from deployment.primitives.device import DeviceSpec +from deployment.evaluation.point_detection_executor import PointDetectionExecutor from deployment.projects.centerpoint.inference.onnx_inference_pipeline import CenterPointONNXInferencePipeline from deployment.projects.centerpoint.inference.pytorch_inference_pipeline import CenterPointPyTorchInferencePipeline from deployment.projects.centerpoint.inference.tensorrt_inference_pipeline import CenterPointTensorRTInferencePipeline -logger = logging.getLogger(__name__) - - -class CenterPointExecutor(BackendExecutor): - """Backend execution primitives for CenterPoint (pipeline creation, input prep). - Args: - components_cfg: Unified components configuration, forwarded to the pipeline - registry when constructing backend pipelines. - """ +class CenterPointExecutor(PointDetectionExecutor): + """Backend execution primitives for CenterPoint (pipeline creation, input prep).""" - def __init__(self, components_cfg: ComponentsConfig) -> None: - super().__init__() - self._components_cfg = components_cfg + task_name = "CenterPoint" + pytorch_pipeline_cls = CenterPointPyTorchInferencePipeline + onnx_pipeline_cls = CenterPointONNXInferencePipeline + tensorrt_pipeline_cls = CenterPointTensorRTInferencePipeline @override def get_output_names(self) -> Optional[List[str]]: """Return the head output names from the components config for verification logging.""" return [out.name for out in self._components_cfg.get_component("pts_backbone_neck_head").io.outputs] - - @override - def create_pipeline(self, model_spec: ModelSpec, device: DeviceSpec) -> BaseInferencePipeline: - """Create a CenterPoint inference pipeline for the given backend and device. - - Args: - model_spec: Model specification (backend, device, path). - device: Target device for the pipeline. - - Returns: - CenterPoint pipeline instance (PyTorch, ONNX, or TensorRT). - - Raises: - ValueError: If ``model_spec.backend`` is not a supported backend. - """ - backend = model_spec.backend - self._validate_backend(backend) - - if backend is Backend.PYTORCH: - logger.info("Creating CenterPoint PyTorch pipeline on %s", device) - return CenterPointPyTorchInferencePipeline(self.pytorch_model, device=device) - - if backend is Backend.ONNX: - logger.info("Creating CenterPoint ONNX pipeline from %s on %s", model_spec.artifact.path, device) - return CenterPointONNXInferencePipeline( - self.pytorch_model, - onnx_dir=model_spec.artifact.path, - device=device, - components_cfg=self._components_cfg, - ) - - if backend is Backend.TENSORRT: - logger.info("Creating CenterPoint TensorRT pipeline from %s on %s", model_spec.artifact.path, device) - return CenterPointTensorRTInferencePipeline( - self.pytorch_model, - tensorrt_dir=model_spec.artifact.path, - device=device, - components_cfg=self._components_cfg, - ) - - raise ValueError(f"Unsupported backend: {backend.value}") - - @override - def prepare_input( - self, - sample: Mapping[str, object], - data_loader: BaseDataLoader, - device: DeviceSpec, - ) -> InferenceInput: - """Build InferenceInput from sample (points + metainfo). - - Args: - sample: Dict with 'points' and 'metainfo'. - data_loader: Unused; kept for interface compatibility. - device: Unused; kept for interface compatibility. - - Returns: - InferenceInput with data=points and metadata=metainfo. - - Raises: - ValueError: If 'points' is missing from sample. - KeyError: If 'metainfo' is missing from sample. - """ - if "points" not in sample: - raise ValueError(f"Expected 'points' in sample. Got keys: {list(sample.keys())}") - if "metainfo" not in sample: - raise KeyError("Sample must contain 'metainfo' for CenterPoint postprocess.") - points = sample["points"] - metadata = sample["metainfo"] - return InferenceInput(data=points, metadata=metadata) diff --git a/deployment/projects/centerpoint/io/model_loader.py b/deployment/projects/centerpoint/io/model_loader.py index 79191deed..39a2ac99f 100644 --- a/deployment/projects/centerpoint/io/model_loader.py +++ b/deployment/projects/centerpoint/io/model_loader.py @@ -7,6 +7,7 @@ from __future__ import annotations import copy +import logging from typing import Tuple import torch @@ -21,6 +22,8 @@ pillar_encoder_onnx, ) +logger = logging.getLogger(__name__) + def create_onnx_model_cfg( model_cfg: Config, @@ -118,5 +121,6 @@ def build_centerpoint_onnx_model( device=device, rot_y_axis_reference=rot_y_axis_reference, ) + model = build_model_from_cfg(export_model_cfg, checkpoint_path, device=device) return model, export_model_cfg diff --git a/deployment/tests/test_centerpoint_configs.py b/deployment/tests/test_centerpoint_configs.py new file mode 100644 index 000000000..e47a64def --- /dev/null +++ b/deployment/tests/test_centerpoint_configs.py @@ -0,0 +1,67 @@ +"""Parse-validation for every CenterPoint deploy config. + +Loads each ``deployment/projects/centerpoint/config/deploy_config*.py`` and feeds its +sections through the typed parsers in ``deployment.config.schema``. This catches +structural/rename errors (e.g. wrong component keys, ``num_warmup_samples`` leftovers, +invalid precision_policy/scenarios) WITHOUT needing a real checkpoint, CUDA, or a full +``BaseDeploymentConfig`` (which validates checkpoint existence + CUDA availability). + +Requires CPU torch + mmengine (the deployment runtime image); no GPU needed. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from mmengine.config import Config + +# Importing the project package registers its ProjectAdapter (required_components). +import deployment.projects.centerpoint # noqa: F401 +from deployment.config.schema import ( + ComponentsConfig, + DeviceConfig, + EvaluationConfig, + ExportConfig, + OnnxConfig, + TensorRTConfig, + VerificationConfig, +) +from deployment.projects.registry import project_registry + +_CONFIG_DIR = Path(__file__).resolve().parents[1] / "projects" / "centerpoint" / "config" +_CONFIG_FILES = sorted(_CONFIG_DIR.glob("deploy_config*.py")) + + +def _config_id(path: Path) -> str: + return path.stem + + +@pytest.mark.parametrize("config_path", _CONFIG_FILES, ids=[_config_id(p) for p in _CONFIG_FILES]) +def test_centerpoint_config_parses(config_path: Path) -> None: + """Every CenterPoint deploy config parses cleanly under the NEW typed schema.""" + cfg = Config.fromfile(str(config_path)) + + # Required sections feed the typed parsers (these raise on structural errors). + components_cfg = ComponentsConfig.from_dict(cfg["components"]) + ExportConfig.from_dict(cfg["export"]) + OnnxConfig.from_dict(cfg.get("onnx_config")) + TensorRTConfig.from_dict(cfg.get("tensorrt_config", {})) + EvaluationConfig.from_dict(cfg.get("evaluation", {})) + VerificationConfig.from_dict(cfg.get("verification", {})) + DeviceConfig.from_dict(cfg.get("devices", {})) + + # Component keys must be the canonical CenterPoint ids (catches un-renamed outer keys). + component_names = set(components_cfg.component_names()) + assert component_names == { + "pts_voxel_encoder", + "pts_backbone_neck_head", + }, f"{config_path.name}: unexpected component keys {sorted(component_names)}" + + # The registry's required-components check is exactly what the entrypoint runs. + project_registry.validate_required_components("centerpoint", components_cfg) + + +def test_config_dir_is_nonempty() -> None: + """Guard against the glob silently matching nothing.""" + assert _CONFIG_FILES, f"No deploy_config*.py found under {_CONFIG_DIR}" diff --git a/projects/BEVFusion/Dockerfile b/projects/BEVFusion/Dockerfile new file mode 100644 index 000000000..4524737f1 --- /dev/null +++ b/projects/BEVFusion/Dockerfile @@ -0,0 +1,59 @@ +ARG AWML_BASE_IMAGE="autoware-ml:latest" +FROM ${AWML_BASE_IMAGE} + +ARG TRT_VERSION=10.8.0.43 +ARG BUILD_SPCONV_CPP="true" +ARG SPCONV_CPP_REF="main" +ARG BUILD_TRT_PLUGINS="true" +# AWML fork of autoware.universe that carries the GetIndicePairsImplicitGemm +# `do_sort` plugin attribute change required by the BEVFusion sparse ONNX export +# (stock upstream autowarefoundation main lacks this attribute). +ARG AUTOWARE_UNIVERSE_REPO="https://github.com/vividf/autoware.universe.git" +ARG AUTOWARE_UNIVERSE_REF="feat/implicit_gemm_int8" + +# Install deployment/runtime dependencies +RUN python3 -m pip --no-cache-dir install \ + onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ \ + onnxsim \ + pycuda \ + tensorrt-cu12==${TRT_VERSION} + +# Install traveller59 python package (requested) +RUN python3 -m pip --no-cache-dir install spconv-cu120 + +# Build and install libspconv.so from spconv_cpp (for C++/TensorRT deployment path) +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + wget && \ + rm -rf /var/lib/apt/lists/* + +RUN if [ "${BUILD_SPCONV_CPP}" = "true" ]; then \ + git clone --depth 1 --branch "${SPCONV_CPP_REF}" https://github.com/autowarefoundation/spconv_cpp.git /opt/spconv_cpp && \ + cd /opt/spconv_cpp && \ + mkdir -p cumm/build-amd64 && \ + cd cumm/build-amd64 && \ + cmake .. && make -j"$(nproc)" && cpack -G DEB && \ + apt-get update && apt-get install -y /opt/spconv_cpp/cumm/_packages/cumm_0.5.3_amd64.deb && \ + cd /opt/spconv_cpp && \ + mkdir -p spconv/build-amd64 && \ + cd spconv/build-amd64 && \ + cmake .. && make -j"$(nproc)" && cpack -G DEB && \ + apt-get update && apt-get install -y /opt/spconv_cpp/spconv/_packages/spconv_2.3.8_amd64.deb && \ + ldconfig; \ + fi + +WORKDIR /workspace +RUN python3 projects/BEVFusion/setup.py develop + +# Build TensorRT custom plugins at image build time. +COPY projects/BEVFusion/plugins/build_plugin_inside_container.sh /workspace/projects/BEVFusion/plugins/build_plugin_inside_container.sh +COPY projects/BEVFusion/plugins/CMakeLists.standalone /workspace/projects/BEVFusion/plugins/CMakeLists.standalone +RUN if [ "${BUILD_TRT_PLUGINS}" = "true" ]; then \ + chmod +x /workspace/projects/BEVFusion/plugins/build_plugin_inside_container.sh && \ + AUTOWARE_UNIVERSE_REPO="${AUTOWARE_UNIVERSE_REPO}" \ + AUTOWARE_UNIVERSE_REF="${AUTOWARE_UNIVERSE_REF}" \ + INSTALL_PLUGINS_DIR="/opt/plugins" \ + bash /workspace/projects/BEVFusion/plugins/build_plugin_inside_container.sh; \ + fi diff --git a/projects/BEVFusion/README.md b/projects/BEVFusion/README.md index 6586b923c..e71de1890 100644 --- a/projects/BEVFusion/README.md +++ b/projects/BEVFusion/README.md @@ -57,6 +57,24 @@ pip install spconv-cu120 `AWML` will automatically select this implementation if the dependency is installed. +#### For ONNX and TensorRT evaluation + +- If you need to use deployment, ONNX runtime, or TensorRT evaluation, please build the docker image first: + +```sh +# Build the base autoware-ml image (if not already built) +DOCKER_BUILDKIT=1 docker build -t autoware-ml . + +# Build the bevfusion-deployment image +DOCKER_BUILDKIT=1 docker build -t bevfusion-deployment -f projects/BEVFusion/Dockerfile . +``` + +- Run the docker container: + +```sh +docker run -it --rm --gpus all --shm-size=64g --name awml_deployment -p 6006:6006 -v $PWD/:/workspace -v $PWD/data:/workspace/data bevfusion-deployment +``` + ### 2. Train #### 2.1. Train the LiDAR-only model first diff --git a/projects/BEVFusion/bevfusion/bevfusion.py b/projects/BEVFusion/bevfusion/bevfusion.py index 709d851a9..75a8a7f72 100644 --- a/projects/BEVFusion/bevfusion/bevfusion.py +++ b/projects/BEVFusion/bevfusion/bevfusion.py @@ -215,6 +215,49 @@ def extract_pts_feat(self, feats, coords, sizes, points=None) -> torch.Tensor: x = self.pts_middle_encoder(feats, coords, batch_size) return x + def _align_lidar_bev_to_head_grid(self, feats): + """Strictly validate pts_backbone+neck BEV maps against head grid resolution. + + ``BEVFusionHead`` builds ``bev_pos`` from ``test_cfg['grid_size'] // out_size_factor`` + (e.g. 1440//8 → 180). Heatmap/top-k indices assume ``H*W == len(bev_pos)``. + + In strict mode, any mismatch is treated as an upstream contract error + (sparse tower stride/layout mismatch) and raises immediately instead of + silently applying pooling. + """ + head = getattr(self, "bbox_head", None) + if head is None or not hasattr(head, "test_cfg") or head.test_cfg is None: + return feats + try: + grid = head.test_cfg["grid_size"] + osf = int(head.test_cfg["out_size_factor"]) + gh = int(grid[0] // osf) + gw = int(grid[1] // osf) + except Exception: + return feats + + def _assert_grid(t: Tensor, *, name: str) -> Tensor: + if t.dim() != 4: + raise AssertionError(f"{name}: expected 4D BEV tensor, got shape={tuple(t.shape)}") + _, _, h, w = t.shape + assert int(h) == gh and int(w) == gw, ( + f"{name}: BEV shape mismatch, got (H,W)=({int(h)},{int(w)}), " + f"expected ({gh},{gw}) from bbox_head.test_cfg grid_size/out_size_factor." + ) + return t + + if isinstance(feats, Tensor): + return _assert_grid(feats, name="lidar_bev") + if isinstance(feats, (list, tuple)): + out = [] + for idx, t in enumerate(feats): + if isinstance(t, Tensor): + out.append(_assert_grid(t, name=f"lidar_bev[{idx}]")) + else: + out.append(t) + return type(feats)(out) + return feats + @torch.no_grad() def voxelize(self, points): feats, coords, sizes = [], [], [] diff --git a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m.py b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m.py index 3edd06c92..25ec70d5e 100644 --- a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m.py +++ b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m.py @@ -13,7 +13,8 @@ # user setting data_root = "data/t4dataset/" -info_directory_path = "info/user_name/" +# info_directory_path = "info/user_name/" +info_directory_path = "info/" experiment_group_name = "bevfusion_lidar_intensity/j6gen2_base/" + _base_.dataset_type experiment_name = "lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m" diff --git a/projects/BEVFusion/plugins/CMakeLists.standalone b/projects/BEVFusion/plugins/CMakeLists.standalone new file mode 100644 index 000000000..2f9324291 --- /dev/null +++ b/projects/BEVFusion/plugins/CMakeLists.standalone @@ -0,0 +1,181 @@ +# Standalone CMakeLists to build autoware_tensorrt_plugins without ament/autoware_cmake. +# Used by build_plugin_inside_container.sh inside BEVFusion Docker. +# Expects: PLUGIN_SRC_DIR, TensorRT_ROOT (e.g. from pip: python -c "import tensorrt; print(tensorrt.__path__[0])") +cmake_minimum_required(VERSION 3.14) +project(autoware_tensorrt_plugins LANGUAGES CXX CUDA) + +set(PLUGIN_SRC_DIR "" CACHE PATH "Path to autoware_tensorrt_plugins source (with src/ and include/)") +set(TensorRT_ROOT "" CACHE PATH "TensorRT root (e.g. pip site-packages/tensorrt)") +set(TensorRT_INCLUDE_DIR "" CACHE PATH "Explicit include directory containing NvInferRuntime.h") +set(TensorRT_EXTRA_HINT_DIRS "" CACHE STRING "Semicolon-separated extra directories to search TensorRT libs") +set(NVINFER_LIBRARY "" CACHE FILEPATH "Explicit path to libnvinfer(.so*)") +set(NVONNXPARSER_LIBRARY "" CACHE FILEPATH "Explicit path to libnvonnxparser(.so*)") + +if(NOT PLUGIN_SRC_DIR OR NOT EXISTS "${PLUGIN_SRC_DIR}/src") + message(FATAL_ERROR "PLUGIN_SRC_DIR must point to the plugin source tree (with src/). Set -DPLUGIN_SRC_DIR=...") +endif() +if(NOT TensorRT_ROOT OR NOT EXISTS "${TensorRT_ROOT}") + message(FATAL_ERROR "TensorRT_ROOT must point to TensorRT (e.g. pip path). Set -DTensorRT_ROOT=...") +endif() + +add_compile_options(-Wno-deprecated-declarations) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CUDA_STANDARD 17) + +# CUDA +find_package(CUDA REQUIRED) +set(CUDA_AVAIL ON) + +# TensorRT: pip wheels may only ship versioned libs (e.g. libnvinfer.so.10) +# and not the unversioned symlink. Support both explicit paths and hint dirs. +if(NOT TensorRT_INCLUDE_DIR) + find_path(TensorRT_INCLUDE_DIR + NAMES NvInferRuntime.h NvInfer.h + HINTS + "${TensorRT_ROOT}" + "${TensorRT_ROOT}/include" + ${TensorRT_EXTRA_HINT_DIRS} + /usr/include + /usr/include/x86_64-linux-gnu + PATH_SUFFIXES + include + include/x86_64-linux-gnu + x86_64-linux-gnu + ) +endif() +if(NOT TensorRT_INCLUDE_DIR) + message(FATAL_ERROR + "TensorRT headers not found (NvInferRuntime.h / NvInfer.h).\n" + " TensorRT_ROOT=${TensorRT_ROOT}\n" + " TensorRT_INCLUDE_DIR=${TensorRT_INCLUDE_DIR}\n" + " TensorRT_EXTRA_HINT_DIRS=${TensorRT_EXTRA_HINT_DIRS}" + ) +endif() +set(TensorRT_INCLUDE_DIRS "${TensorRT_INCLUDE_DIR}") + +set(TRT_HINT_DIRS "${TensorRT_ROOT}" "${TensorRT_ROOT}/lib") +if(TensorRT_EXTRA_HINT_DIRS) + list(APPEND TRT_HINT_DIRS ${TensorRT_EXTRA_HINT_DIRS}) +endif() +list(REMOVE_DUPLICATES TRT_HINT_DIRS) + +if(NOT NVINFER_LIBRARY) + find_library(NVINFER_LIBRARY + NAMES + nvinfer + nvinfer.so.10 nvinfer.so.9 nvinfer.so.8 + libnvinfer.so.10 libnvinfer.so.9 libnvinfer.so.8 + HINTS ${TRT_HINT_DIRS} + ) +endif() +if(NOT NVONNXPARSER_LIBRARY) + find_library(NVONNXPARSER_LIBRARY + NAMES + nvonnxparser + nvonnxparser.so.10 nvonnxparser.so.9 nvonnxparser.so.8 + libnvonnxparser.so.10 libnvonnxparser.so.9 libnvonnxparser.so.8 + HINTS ${TRT_HINT_DIRS} + ) +endif() +if(NOT NVINFER_LIBRARY OR NOT NVONNXPARSER_LIBRARY) + message(FATAL_ERROR + "TensorRT libraries not found.\n" + " NVINFER_LIBRARY=${NVINFER_LIBRARY}\n" + " NVONNXPARSER_LIBRARY=${NVONNXPARSER_LIBRARY}\n" + " TensorRT_ROOT=${TensorRT_ROOT}\n" + " TensorRT_EXTRA_HINT_DIRS=${TensorRT_EXTRA_HINT_DIRS}\n" + " TRT_HINT_DIRS=${TRT_HINT_DIRS}" + ) +endif() +message(STATUS "TensorRT NVINFER: ${NVINFER_LIBRARY}") +message(STATUS "TensorRT NVONNXPARSER: ${NVONNXPARSER_LIBRARY}") +message(STATUS "TensorRT includes: ${TensorRT_INCLUDE_DIRS}") +set(TRT_AVAIL ON) + +# spconv / cumm (from .deb installed in container) +find_package(cumm QUIET) +find_package(spconv QUIET) +if(NOT cumm_FOUND OR NOT spconv_FOUND) + message(FATAL_ERROR "cumm and spconv required (install spconv_cpp .deb first). cumm_FOUND=${cumm_FOUND} spconv_FOUND=${spconv_FOUND}") +endif() +set(SPCONV_AVAIL ON) + +include_directories( + "${PLUGIN_SRC_DIR}/include" + "${PLUGIN_SRC_DIR}/src" + ${CUDA_INCLUDE_DIRS} + ${TensorRT_INCLUDE_DIRS} +) +add_definitions(-DTV_CUDA) + +# CUDA compile options (must be attached to targets so nvcc sees them) +set(TRT_PLUGIN_CUDA_FLAGS + "--expt-relaxed-constexpr" + "--extended-lambda" + "-diag-suppress=1675" + "-gencode=arch=compute_75,code=sm_75" + "-gencode=arch=compute_86,code=sm_86" + "-gencode=arch=compute_87,code=sm_87" + "-gencode=arch=compute_89,code=sm_89" +) +if(CUDA_VERSION VERSION_GREATER_EQUAL "12.8") + list(APPEND TRT_PLUGIN_CUDA_FLAGS "-gencode=arch=compute_89,code=compute_89") +endif() + +# cuda_ops (CUDA shared library; container has CUDA from base image) +add_library(cuda_ops SHARED + "${PLUGIN_SRC_DIR}/src/argsort_ops/argsort.cu" + "${PLUGIN_SRC_DIR}/src/bev_ops/bev_pool_cuda.cu" + "${PLUGIN_SRC_DIR}/src/scatter_ops/segment_csr.cu" + "${PLUGIN_SRC_DIR}/src/unique_ops/unique.cu" + "${PLUGIN_SRC_DIR}/src/multi_scale_deform_attn_ops/ms_deform_attn_kernel.cu" + "${PLUGIN_SRC_DIR}/src/rotate_ops/rotate_kernel.cu" + "${PLUGIN_SRC_DIR}/src/select_and_pad_ops/select_and_pad_kernel.cu" +) +target_include_directories(cuda_ops PRIVATE ${CUDA_INCLUDE_DIRS} "${PLUGIN_SRC_DIR}/include" "${PLUGIN_SRC_DIR}/src") +set_target_properties(cuda_ops PROPERTIES + POSITION_INDEPENDENT_CODE ON + CUDA_SEPARABLE_COMPILATION ON +) +target_compile_options(cuda_ops PRIVATE + $<$:${TRT_PLUGIN_CUDA_FLAGS}> +) + +# Main plugin library +add_library(${PROJECT_NAME} SHARED + "${PLUGIN_SRC_DIR}/src/argsort_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/argsort_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/quick_cumsum_cuda_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/quick_cumsum_cuda_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/get_indices_pairs_implicit_gemm_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/get_indices_pairs_implicit_gemm_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/get_indices_pairs_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/get_indices_pairs_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/implicit_gemm_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/implicit_gemm_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/indice_conv_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/indice_conv_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/multi_scale_deformable_attention_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/multi_scale_deformable_attention_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/rotate_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/rotate_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/segment_csr_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/segment_csr_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/select_and_pad_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/select_and_pad_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/unique_plugin.cpp" + "${PLUGIN_SRC_DIR}/src/unique_plugin_creator.cpp" + "${PLUGIN_SRC_DIR}/src/plugin_registration.cpp" + "${PLUGIN_SRC_DIR}/src/plugin_utils.cpp" +) +target_compile_definitions(${PROJECT_NAME} PRIVATE _GLIBCXX_USE_CXX11_ABI=1) +target_compile_options(${PROJECT_NAME} PRIVATE + $<$:${TRT_PLUGIN_CUDA_FLAGS}> +) +target_link_libraries(${PROJECT_NAME} PRIVATE + ${NVINFER_LIBRARY} + ${NVONNXPARSER_LIBRARY} + CUDA::cudart + cuda_ops + spconv::spconv +) diff --git a/projects/BEVFusion/plugins/README.md b/projects/BEVFusion/plugins/README.md new file mode 100644 index 000000000..d2166fa6b --- /dev/null +++ b/projects/BEVFusion/plugins/README.md @@ -0,0 +1,95 @@ +# Place TensorRT Plugin `.so` Here + +Put your TensorRT custom plugin shared library in this directory **before** building the BEVFusion Docker image. + +## Why you need this (ImplicitGemm / "Plugin not found") + +The BEVFusion ONNX model contains **sparse convolution** nodes (`ImplicitGemm`, `GetIndicePairsImplicitGemm`). TensorRT does not ship these; they are provided by a **custom plugin** that must be loaded before parsing ONNX. If you see: + +```text +Plugin not found, are the plugin name, version, and namespace correct? +operator: ImplicitGemm (checkFallbackPluginImporter) +``` + +then the container is **missing the plugin .so** or it is not being loaded (see config below). + +## Where to get the plugin .so + +The plugin is **not** built by `spconv_cpp` (that repo only builds `libspconv.so` and cumm). The TensorRT plugin that registers `ImplicitGemm` and `GetIndicePairsImplicitGemm` is built from **Autoware**'s package: + +- **Repository:** [autoware_universe / perception / autoware_tensorrt_plugins](https://github.com/autowarefoundation/autoware_universe/tree/main/perception/autoware_tensorrt_plugins) +- **Build requirements:** TensorRT (e.g. 10.x), CUDA, and **spconv** (cumm + libspconv) installed so `find_package(spconv)` succeeds. + +**Option A – Build from Autoware (recommended for version match)** +In an environment with the same TensorRT/CUDA as your BEVFusion image (e.g. TensorRT 10.8, CUDA 12): + +1. Clone `autoware_universe`, install cumm + spconv (e.g. from spconv_cpp .deb or build). +2. Build the `autoware_tensorrt_plugins` package (e.g. with colcon). +3. Copy the built shared library (e.g. `libautoware_tensorrt_plugins.so` from the install tree) into this directory: + `AWML/projects/BEVFusion/plugins/` +4. Rebuild the BEVFusion Docker image so the Dockerfile `COPY projects/BEVFusion/plugins/ /opt/plugins/` includes the .so. + +**Option B – Copy from an Autoware Docker image** +If you have an Autoware image that already builds perception (e.g. `universe-sensing-perception-devel-cuda`): + +```bash +# Find the plugin in the Autoware container (path may vary by install layout) +docker run --rm autoware:universe-sensing-perception-devel-cuda find /opt -name "*tensorrt*plugin*.so" 2>/dev/null +# Copy it out and put the file into projects/BEVFusion/plugins/ +docker create --name tmp-autoware autoware:universe-sensing-perception-devel-cuda +docker cp tmp-autoware:/path/to/libautoware_tensorrt_plugins.so ./projects/BEVFusion/plugins/ +docker rm tmp-autoware +``` + +Then rebuild the BEVFusion image. + +**Option C – Build inside the BEVFusion container** +You can compile the plugin **inside** your existing BEVFusion Docker (no host build, no second image). The image already has TensorRT (pip), CUDA, and spconv/cumm from the Dockerfile’s spconv_cpp build. + +```bash +# Enter your BEVFusion container +docker run -it --rm --gpus all -v $PWD:/workspace -w /workspace awml-bevfusion:full bash + +# Build the plugin (clones Autoware plugin source, builds with CMake, installs to /opt/plugins/) +bash projects/BEVFusion/plugins/build_plugin_inside_container.sh + +# Then set config or env and run export again (see “Config” below) +``` + +- Script location: `projects/BEVFusion/plugins/build_plugin_inside_container.sh` +- It clones `autoware_universe` (sparse checkout, plugin only), builds with a standalone CMakeLists (no ament/ROS2), and copies `libautoware_tensorrt_plugins.so` to `/opt/plugins/`. +- Optional env vars: `BUILD_DIR`, `SRC_DIR`, `INSTALL_PLUGINS_DIR`, `AUTOWARE_UNIVERSE_REF` (default `main`). +- If build fails on `find_package(spconv)` / `find_package(cumm)`, ensure the image was built with `BUILD_SPCONV_CPP=true` (Dockerfile default) so the spconv .deb is installed. + +## Expected filename and paths + +- In this folder (host): `libautoware_tensorrt_plugins.so` +- After Docker build (in container): `/opt/plugins/libautoware_tensorrt_plugins.so` + +## Config: tell AWML to load the plugin + +- **File:** `deployment/projects/bevfusion/config/deploy_config.py` +- Set `plugin_libraries` so the exporter loads the .so **before** parsing ONNX: + +```python +tensorrt_config = dict( + precision_policy="auto", + max_workspace_size=1 << 32, + plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"], +) +``` + +Alternatively, set the env var (no image rebuild needed if the .so is already in the container): + +```bash +export DEPLOY_TENSORRT_PLUGIN_LIBS=/opt/plugins/libautoware_tensorrt_plugins.so +``` + +## Quick check inside container + +```bash +python projects/BEVFusion/deploy/check_trt_spconv_plugins.py \ + --plugin-so /opt/plugins/libautoware_tensorrt_plugins.so +``` + +You should see `ImplicitGemm` and `GetIndicePairsImplicitGemm` in the plugin creator list. diff --git a/projects/BEVFusion/plugins/build_plugin_inside_container.sh b/projects/BEVFusion/plugins/build_plugin_inside_container.sh new file mode 100644 index 000000000..3c0d9a8a8 --- /dev/null +++ b/projects/BEVFusion/plugins/build_plugin_inside_container.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# Build libautoware_tensorrt_plugins.so inside the BEVFusion Docker container. +# Run this script from inside the container (e.g. /workspace or any dir). +# Usage: bash projects/BEVFusion/plugins/build_plugin_inside_container.sh +# Local source (no clone): AUTOWARE_TENSORRT_PLUGINS_SRC=/path/to/perception/autoware_tensorrt_plugins +# Headers: must match the pip TensorRT runtime (libnvinfer.so.10). The script resolves them in +# order: (1) TensorRT_INCLUDE_DIR env override, (2) pip/site-packages headers IF their +# major.minor matches the runtime, (3) fetch matching public headers from NVIDIA/TensorRT OSS +# (release/). It deliberately does NOT apt-install libnvinfer-dev — Ubuntu's +# TensorRT is a different major.minor and causes an ABI/vtable segfault at plugin createPlugin. +# Offline? Pre-fetch headers and pass TensorRT_INCLUDE_DIR=/path/to/TensorRT-/include. +# Result: libautoware_tensorrt_plugins.so is written to /opt/plugins/ +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +BUILD_DIR="${BUILD_DIR:-/tmp/trt_plugin_build}" +SRC_DIR="${SRC_DIR:-/tmp/autoware_tensorrt_plugins_src}" +INSTALL_PLUGINS_DIR="${INSTALL_PLUGINS_DIR:-/opt/plugins}" +# AWML clones autoware_tensorrt_plugins from an AWML-maintained fork that bakes +# in the GetIndicePairsImplicitGemm do_sort-attribute change (required by the +# BEVFusion sparse ONNX export). +# Override the URL/ref via env vars when you want to track a different fork/branch +# (e.g. upstream autowarefoundation/autoware.universe main for an A/B build). +AUTOWARE_UNIVERSE_REPO="${AUTOWARE_UNIVERSE_REPO:-https://github.com/vividf/autoware.universe.git}" +# AUTOWARE_UNIVERSE_REF="${AUTOWARE_UNIVERSE_REF:-feat/spconv-do-sort-attribute}" +AUTOWARE_UNIVERSE_REF="${AUTOWARE_UNIVERSE_REF:-feat/implicit_gemm_int8}" + +echo "[build_plugin] Script dir: $SCRIPT_DIR" +echo "[build_plugin] Build dir: $BUILD_DIR" +echo "[build_plugin] Source dir (clone): $SRC_DIR" +echo "[build_plugin] Install .so to: $INSTALL_PLUGINS_DIR" +echo "[build_plugin] Plugin repo: $AUTOWARE_UNIVERSE_REPO" +echo "[build_plugin] Plugin ref: $AUTOWARE_UNIVERSE_REF" + +# Resolve TensorRT from pip so CMake can find headers/libs +if python3 -c "import tensorrt" 2>/dev/null; then + TRT_PIP_PATH="$(python3 -c "import tensorrt; print(tensorrt.__path__[0])")" + echo "[build_plugin] TensorRT (pip): $TRT_PIP_PATH" +else + echo "[build_plugin] ERROR: TensorRT not found (pip). Install with: pip install tensorrt-cu12" + exit 1 +fi + +# Ensure LD_LIBRARY_PATH includes TensorRT libs from pip when we build/run +export LD_LIBRARY_PATH="${TRT_PIP_PATH}:${LD_LIBRARY_PATH}" + +# Discover TensorRT lib directories/files from pip installation. +TRT_DISCOVERY="$( +python3 - <<'PY' +import glob +import os +import site + +paths = [] +for p in site.getsitepackages() + [site.getusersitepackages()]: + if p and os.path.isdir(p): + paths.append(p) + +candidates = [] +for base in paths: + for rel in ( + "tensorrt", + "tensorrt/lib", + "tensorrt_libs", + "nvidia/tensorrt", + "nvidia/tensorrt/lib", + ): + c = os.path.join(base, rel) + if os.path.isdir(c): + candidates.append(c) + +def pick(patterns): + for d in candidates: + for pat in patterns: + matches = sorted(glob.glob(os.path.join(d, pat))) + if matches: + return matches[0] + return "" + +nv = pick(["libnvinfer.so*", "nvinfer.so*"]) +onnx = pick(["libnvonnxparser.so*", "nvonnxparser.so*"]) + +include_candidates = [] +for d in candidates: + for rel in ("include", ""): + t = os.path.join(d, rel, "NvInferRuntime.h") + if os.path.isfile(t): + include_candidates.append(os.path.dirname(t)) +for d in ("/usr/include", "/usr/include/x86_64-linux-gnu"): + t = os.path.join(d, "NvInferRuntime.h") + if os.path.isfile(t): + include_candidates.append(os.path.dirname(t)) + +print("HINTS=" + ";".join(dict.fromkeys(candidates))) +print("NVINFER=" + nv) +print("NVONNXPARSER=" + onnx) +print("INCLUDE=" + (include_candidates[0] if include_candidates else "")) +PY +)" +TRT_HINT_DIRS="$(echo "$TRT_DISCOVERY" | awk -F= '/^HINTS=/{print $2}')" +NVINFER_LIB="$(echo "$TRT_DISCOVERY" | awk -F= '/^NVINFER=/{print $2}')" +NVONNXPARSER_LIB="$(echo "$TRT_DISCOVERY" | awk -F= '/^NVONNXPARSER=/{print $2}')" +DISCOVERED_INCLUDE="$(echo "$TRT_DISCOVERY" | awk -F= '/^INCLUDE=/{print $2}')" +echo "[build_plugin] TensorRT hint dirs: ${TRT_HINT_DIRS:-}" +echo "[build_plugin] NVINFER candidate: ${NVINFER_LIB:-}" +echo "[build_plugin] NVONNXPARSER candidate: ${NVONNXPARSER_LIB:-}" +echo "[build_plugin] Discovered include candidate: ${DISCOVERED_INCLUDE:-}" + +# --------------------------------------------------------------------------- +# Resolve TensorRT C++ headers that MATCH the pip runtime (libnvinfer.so.10). +# +# Why this matters: the pip wheel ships runtime libs but usually NO C++ headers, +# and Ubuntu's apt `libnvinfer-dev` is a DIFFERENT TensorRT major.minor (e.g. 11.x). +# Building the plugin against mismatched headers while linking the pip libs produces +# an IPluginCreatorV3One vtable/ABI mismatch that SEGFAULTS at plugin createPlugin +# time during ONNX parse. So we require headers whose major.minor == the runtime's, +# and we DO NOT apt-install libnvinfer-dev. +# --------------------------------------------------------------------------- +TRT_PY_VERSION="$(python3 -c 'import tensorrt as t; print(t.__version__)' 2>/dev/null)" +TRT_MM="$(echo "$TRT_PY_VERSION" | awk -F. 'NF>=2{print $1"."$2}')" +echo "[build_plugin] pip TensorRT runtime: ${TRT_PY_VERSION:-} (require headers ${TRT_MM:-?})" + +# Echo the major.minor recorded in a header dir's NvInferVersion.h, or empty. +# Note: a mismatched system header (e.g. Ubuntu's) may use non-numeric macros +# (TRT_*_ENTERPRISE) -> returns empty -> correctly rejected below. +_hdr_mm() { + local h="$1/NvInferVersion.h" maj min + [ -f "$h" ] || { echo ""; return; } + maj="$(awk '/#define[ \t]+NV_TENSORRT_MAJOR/{print $3; exit}' "$h")" + min="$(awk '/#define[ \t]+NV_TENSORRT_MINOR/{print $3; exit}' "$h")" + case "$maj" in ''|*[!0-9]*) echo ""; return;; esac + case "$min" in ''|*[!0-9]*) echo ""; return;; esac + echo "${maj}.${min}" +} + +TRT_INCLUDE_DIR="" +# (1) Explicit override wins (must contain NvInferRuntime.h). +if [ -n "${TensorRT_INCLUDE_DIR:-}" ] && [ -f "${TensorRT_INCLUDE_DIR}/NvInferRuntime.h" ]; then + TRT_INCLUDE_DIR="$TensorRT_INCLUDE_DIR" + echo "[build_plugin] Using TensorRT_INCLUDE_DIR override: $TRT_INCLUDE_DIR (mm=$(_hdr_mm "$TRT_INCLUDE_DIR"))" +fi +# (2) Discovered headers — only if their version matches the pip runtime. +if [ -z "$TRT_INCLUDE_DIR" ] && [ -n "$DISCOVERED_INCLUDE" ]; then + cand_mm="$(_hdr_mm "$DISCOVERED_INCLUDE")" + if [ -n "$TRT_MM" ] && [ "$cand_mm" = "$TRT_MM" ]; then + TRT_INCLUDE_DIR="$DISCOVERED_INCLUDE" + echo "[build_plugin] Using discovered TensorRT headers: $TRT_INCLUDE_DIR (mm=$cand_mm)" + else + echo "[build_plugin] Ignoring discovered headers $DISCOVERED_INCLUDE (version '${cand_mm:-?}' != runtime '${TRT_MM:-?}') to avoid an ABI mismatch." + fi +fi +# (3) Fetch version-matched PUBLIC headers from TensorRT OSS (NOT apt). +if [ -z "$TRT_INCLUDE_DIR" ]; then + if [ -z "$TRT_MM" ]; then + echo "[build_plugin] ERROR: cannot determine the pip TensorRT version; set TensorRT_INCLUDE_DIR explicitly." + exit 1 + fi + OSS_DIR="${TRT_OSS_HEADERS_DIR:-/tmp/tensorrt_oss_headers}" + if [ ! -f "$OSS_DIR/include/NvInferRuntime.h" ]; then + echo "[build_plugin] Fetching TensorRT $TRT_MM public headers from NVIDIA/TensorRT (release/$TRT_MM)..." + rm -rf "$OSS_DIR" + git clone -q -b "release/$TRT_MM" --depth 1 --filter=blob:none --sparse \ + https://github.com/NVIDIA/TensorRT.git "$OSS_DIR" \ + && ( cd "$OSS_DIR" && git sparse-checkout set include >/dev/null 2>&1 ) || true + fi + oss_mm="$(_hdr_mm "$OSS_DIR/include")" + if [ "$oss_mm" = "$TRT_MM" ]; then + TRT_INCLUDE_DIR="$OSS_DIR/include" + echo "[build_plugin] Using TensorRT OSS headers: $TRT_INCLUDE_DIR (mm=$oss_mm)" + fi +fi + +if [ -z "$TRT_INCLUDE_DIR" ]; then + echo "[build_plugin] ERROR: could not obtain TensorRT C++ headers matching runtime ${TRT_MM:-?}." + echo "[build_plugin] - point TensorRT_INCLUDE_DIR at TensorRT-${TRT_PY_VERSION:-}/include, or" + echo "[build_plugin] - ensure network access to github.com/NVIDIA/TensorRT (release/${TRT_MM:-?})." + echo "[build_plugin] DO NOT use apt 'libnvinfer-dev': Ubuntu ships a different TensorRT major.minor," + echo "[build_plugin] which links against the pip libs and segfaults at plugin createPlugin (ABI mismatch)." + exit 1 +fi +echo "[build_plugin] TensorRT include dir (final): $TRT_INCLUDE_DIR" + +# Optional: use a bind-mounted `perception/autoware_tensorrt_plugins` tree and skip `git clone`. +# Example: export AUTOWARE_TENSORRT_PLUGINS_SRC=/workspace/autoware.universe/perception/autoware_tensorrt_plugins +AUTOWARE_TENSORRT_PLUGINS_SRC="${AUTOWARE_TENSORRT_PLUGINS_SRC:-}" +PLUGIN_SRC_DIR="" +if [ -n "$AUTOWARE_TENSORRT_PLUGINS_SRC" ]; then + if [ -f "$AUTOWARE_TENSORRT_PLUGINS_SRC/src/implicit_gemm_plugin.cpp" ]; then + PLUGIN_SRC_DIR="$(cd "$AUTOWARE_TENSORRT_PLUGINS_SRC" && pwd)" + echo "[build_plugin] Using AUTOWARE_TENSORRT_PLUGINS_SRC=$PLUGIN_SRC_DIR (skipping git clone)" + else + echo "[build_plugin] ERROR: AUTOWARE_TENSORRT_PLUGINS_SRC=$AUTOWARE_TENSORRT_PLUGINS_SRC" + echo "[build_plugin] must contain src/implicit_gemm_plugin.cpp" + exit 1 + fi +fi + +# Clone only perception/autoware_tensorrt_plugins from the configured fork. +# The fork (default: vividf/autoware.universe @ feat/spconv-do-sort-attribute) +# already contains the `do_sort` attribute change; no source patching here. +# To A/B against stock upstream, set: +# AUTOWARE_UNIVERSE_REPO=https://github.com/autowarefoundation/autoware.universe.git +# AUTOWARE_UNIVERSE_REF=main +# +# Cache invalidation: if $SRC_DIR exists but was cloned from a different +# repo/ref (common after switching to an AWML fork), we must re-clone, otherwise +# we silently build stale upstream source. The repo/ref pair is recorded in +# $SRC_DIR/.awml_clone_meta and compared on every invocation. +if [ -z "$PLUGIN_SRC_DIR" ]; then + CLONE_META_FILE="$SRC_DIR/.awml_clone_meta" + EXPECTED_META="${AUTOWARE_UNIVERSE_REPO}@${AUTOWARE_UNIVERSE_REF}" + NEEDS_CLONE=0 + if [ ! -d "$SRC_DIR/src" ] && [ ! -d "$SRC_DIR/perception/autoware_tensorrt_plugins/src" ]; then + NEEDS_CLONE=1 + elif [ ! -f "$CLONE_META_FILE" ] || [ "$(cat "$CLONE_META_FILE" 2>/dev/null)" != "$EXPECTED_META" ]; then + echo "[build_plugin] Cached clone at $SRC_DIR does not match $EXPECTED_META; forcing re-clone." + NEEDS_CLONE=1 + fi + + if [ "$NEEDS_CLONE" = "1" ]; then + echo "[build_plugin] Cloning autoware_tensorrt_plugins source from $AUTOWARE_UNIVERSE_REPO @ $AUTOWARE_UNIVERSE_REF ..." + rm -rf "$SRC_DIR" + git clone --depth 1 --branch "$AUTOWARE_UNIVERSE_REF" \ + --filter=blob:none --sparse \ + "$AUTOWARE_UNIVERSE_REPO" "$SRC_DIR" + (cd "$SRC_DIR" && git sparse-checkout set perception/autoware_tensorrt_plugins) + echo "$EXPECTED_META" > "$CLONE_META_FILE" + fi + if [ -d "$SRC_DIR/perception/autoware_tensorrt_plugins" ]; then + PLUGIN_SRC_DIR="$SRC_DIR/perception/autoware_tensorrt_plugins" + else + PLUGIN_SRC_DIR="$SRC_DIR" + fi +fi + +if [ ! -f "$PLUGIN_SRC_DIR/src/implicit_gemm_plugin.cpp" ]; then + echo "[build_plugin] ERROR: Clone failed or layout changed; $PLUGIN_SRC_DIR/src/implicit_gemm_plugin.cpp not found" + exit 1 +fi + +# Sanity-check that the fork actually carries the do_sort attribute change. +# If the user overrode AUTOWARE_UNIVERSE_REPO/REF to stock upstream, this will +# warn but not fail (intended for A/B builds). +if grep -q "\"do_sort\"" "$PLUGIN_SRC_DIR/src/get_indices_pairs_implicit_gemm_plugin_creator.cpp" 2>/dev/null; then + echo "[build_plugin] OK: cloned source exposes do_sort plugin attribute" +else + echo "[build_plugin] WARNING: cloned source does NOT expose the do_sort attribute." + echo "[build_plugin] This is fine ONLY if you are intentionally A/B-testing against stock upstream." + echo "[build_plugin] For the BEVFusion sparse export, point AUTOWARE_UNIVERSE_REPO/REF at the AWML fork." +fi + +# Configure and build with standalone CMakeLists (no ament/autoware_cmake) +mkdir -p "$BUILD_DIR" +cp "$SCRIPT_DIR/CMakeLists.standalone" "$BUILD_DIR/CMakeLists.txt" +cd "$BUILD_DIR" +rm -f CMakeCache.txt +cmake . \ + -DPLUGIN_SRC_DIR="$PLUGIN_SRC_DIR" \ + -DTensorRT_ROOT="$TRT_PIP_PATH" \ + -DTensorRT_EXTRA_HINT_DIRS="$TRT_HINT_DIRS" \ + -DTensorRT_INCLUDE_DIR="$TRT_INCLUDE_DIR" \ + -DNVINFER_LIBRARY="$NVINFER_LIB" \ + -DNVONNXPARSER_LIBRARY="$NVONNXPARSER_LIB" \ + -DCMAKE_BUILD_TYPE=Release +make -j"$(nproc)" + +SO_NAME="libautoware_tensorrt_plugins.so" +if [ ! -f "$BUILD_DIR/$SO_NAME" ]; then + echo "[build_plugin] ERROR: Build did not produce $SO_NAME" + exit 1 +fi + +mkdir -p "$INSTALL_PLUGINS_DIR" +cp -a "$BUILD_DIR/$SO_NAME" "$INSTALL_PLUGINS_DIR/" +chmod 755 "$INSTALL_PLUGINS_DIR/$SO_NAME" +echo "[build_plugin] Installed: $INSTALL_PLUGINS_DIR/$SO_NAME" +ldconfig 2>/dev/null || true +echo "[build_plugin] Done. Set plugin_libraries=[\"$INSTALL_PLUGINS_DIR/$SO_NAME\"] in deploy_config.py or export DEPLOY_TENSORRT_PLUGIN_LIBS=$INSTALL_PLUGINS_DIR/$SO_NAME" +echo "[build_plugin] Verify: python3 projects/BEVFusion/deploy/check_trt_spconv_plugins.py --plugin-so $INSTALL_PLUGINS_DIR/$SO_NAME" diff --git a/projects/SparseConvolution/sparse_conv.py b/projects/SparseConvolution/sparse_conv.py index d4ed1d38e..bbed9b8fc 100644 --- a/projects/SparseConvolution/sparse_conv.py +++ b/projects/SparseConvolution/sparse_conv.py @@ -63,10 +63,8 @@ def _conv_forward( act_beta: float = 0, ): # assert isinstance(input, SparseConvTensor) - is_int8 = input.is_quantized and weight.is_quantized - if is_int8: - raise NotImplementedError - + # INT8 path is not implemented below (raises NotImplementedError); keep FP32/dequant path only. + is_int8 = False assert input.features.shape[1] == self.in_channels, "channel size mismatch" features = input.features indices = input.indices @@ -75,8 +73,6 @@ def _conv_forward( bias_for_training = bias if training else None bias_for_infer = bias if not training else None output_add_scale = 0.0 - if is_int8: - raise NotImplementedError if training: raise NotImplementedError @@ -102,9 +98,13 @@ def _conv_forward( raise NotImplementedError indice_dict = input.indice_dict.copy() - # only support contiguous tensor for now + # only support contiguous tensor for now (indices non-contiguous → CUDA illegal access in merge_sort) if not features.is_contiguous(): features = features.contiguous() + if not indices.is_contiguous(): + indices = indices.contiguous() + if indices.dtype != torch.int32: + indices = indices.to(dtype=torch.int32) algo = self.algo if self.indice_key is not None: data = input.find_indice_pair(self.indice_key) @@ -257,7 +257,8 @@ def _conv_forward( if input.benchmark: torch.cuda.synchronize() t = time.time() - with input._timer.namespace("gen_pairs"): + _gen_ctx = input._timer.namespace("gen_pairs") if input._timer is not None else nullcontext() + with _gen_ctx: # we need to gen bwd indices for regular conv # because it may be inversed. try: @@ -280,11 +281,22 @@ def _conv_forward( ) ) except Exception as e: - msg = "[Exception|implicit_gemm_pair]" - msg += f"indices={indices.shape}," "bs={batch_size}," "ss={spatial_shape}," - msg += f"algo={algo}," "ksize={self.kernel_size}," "stride={self.stride}," - msg += f"padding={self.padding}," "dilation={self.dilation}," "subm={self.subm}," - msg += f"transpose={self.transposed}" + + def _idx_shape_str() -> str: + try: + return str(tuple(int(s) for s in indices.shape)) + except Exception: + return str(indices.shape) + + msg = ( + f"[Exception|implicit_gemm_pair]indices_shape={_idx_shape_str()}," + f"bs={batch_size},ss={spatial_shape},algo={algo}," + f"ksize={self.kernel_size},stride={self.stride}," + f"padding={self.padding},dilation={self.dilation}," + f"subm={self.subm},transpose={self.transposed}," + f"indices_dtype={getattr(indices, 'dtype', None)}," + f"indices_device={getattr(indices, 'device', None)}" + ) print(msg, file=sys.stderr) spconv_save_debug_data(indices) raise e @@ -351,7 +363,7 @@ def _conv_forward( self.subm, input._timer, self.fp32_accum, - None, + bias_cur, act_alpha, act_beta, act_type, @@ -364,9 +376,6 @@ def _conv_forward( output_dtype, ) - if bias_cur is not None: - out_features = out_features + bias_cur - if bias_for_training is not None: out_features += bias_for_training if input.benchmark: diff --git a/projects/SparseConvolution/sparse_functional.py b/projects/SparseConvolution/sparse_functional.py index efb72f7e1..f5930d0eb 100644 --- a/projects/SparseConvolution/sparse_functional.py +++ b/projects/SparseConvolution/sparse_functional.py @@ -6,7 +6,7 @@ from cumm import tensorview as tv from spconv import constants from spconv.algo import CONV_CPP -from spconv.constants import SPCONV_DO_SORT, SPCONV_USE_DIRECT_TABLE, AllocKeys +from spconv.constants import SPCONV_USE_DIRECT_TABLE, AllocKeys from spconv.core import ConvAlgo from spconv.core_cc.csrc.sparse.all import SpconvOps from spconv.core_cc.csrc.sparse.convops.spops import ConvGemmOps @@ -17,6 +17,34 @@ from torch.autograd import Function from torch.onnx.symbolic_helper import _get_tensor_sizes +# Controls `do_sort` on GetIndicePairsImplicitGemm (ONNX export + PyTorch +# forward). Default True (pair-mask argsort on). Deploy CLIs (e.g. the BEVFusion +# entrypoint) may flip this to False via `set_do_sort` before ONNX export. There +# is deliberately no env-var fallback: the deploy_config is the single source of truth. +_do_sort: bool = True + + +def set_do_sort(value: bool) -> None: + """Set ``do_sort`` used at ONNX export and in the PyTorch forward path for + pair-mask argsort. Called by deploy CLIs from ``deploy_cfg.spconv_do_sort``.""" + global _do_sort + _do_sort = bool(value) + + +def _gemm_activation_to_onnx_int(act_type: Any) -> int: + """Map ``tv.gemm.Activation`` to its integer value (see cumm ``constants.h``).""" + if act_type is None: + return 0 + if isinstance(act_type, int): + return int(act_type) + v = getattr(act_type, "value", None) + if v is not None: + return int(v) + try: + return int(act_type) + except Exception: + return 0 + class GetIndicePairs(Function): @@ -107,7 +135,7 @@ def forward( pair = alloc.allocated[AllocKeys.PairFwd] indice_num_per_loc = alloc.allocated[AllocKeys.IndiceNumPerLoc] - num_act_out = torch.tensor([num_act_out], dtype=torch.int32).to(out_inds.device) + num_act_out = torch.tensor([num_act_out], dtype=torch.int32, device=out_inds.device) return out_inds[:num_act_out], pair, indice_num_per_loc, num_act_out @@ -212,7 +240,6 @@ def backward(ctx: Any, grad_output: torch.Tensor) -> tuple: class GetIndicePairsImplicitGemm(Function): - @staticmethod def symbolic( g, @@ -245,6 +272,7 @@ def symbolic( subm_i=subm, transpose_i=transpose, is_train_i=is_train, + do_sort_i=int(_do_sort), outputs=5, ) indices_shape = _get_tensor_sizes(indices) @@ -301,7 +329,7 @@ def forward( num_out_act_bound: int = -1 direct_table: bool = SPCONV_USE_DIRECT_TABLE - do_sort = SPCONV_DO_SORT + do_sort = _do_sort stream = get_current_stream() @@ -331,7 +359,7 @@ def forward( do_sort=do_sort, ) - num_act_out = torch.tensor([num_act_out], dtype=torch.int32).to(indices.device) + num_act_out = torch.tensor([num_act_out], dtype=torch.int32, device=indices.device) mask_split_count = mask_tensor.dim(0) # NOTE(knzo25): we support only the simplest case @@ -389,19 +417,20 @@ def symbolic( output_add_scale: float, output_dtype: Optional[torch.dtype], ): + gemm_inputs = [features, filters, pair_fwd, pair_mask_fwd_splits, mask_argsort_fwd_splits] + if bias is not None: + # Optional 6th input for folded per-channel bias (C_out). + gemm_inputs.append(bias) output = g.op( "autoware::ImplicitGemm", - features, - filters, - pair_fwd, - pair_mask_fwd_splits, - mask_argsort_fwd_splits, + *gemm_inputs, is_train_i=is_train, is_subm_i=is_subm, fp32_accum_i=fp32_accum, act_alpha_f=act_alpha, act_beta_f=act_beta, + act_type_i=_gemm_activation_to_onnx_int(act_type), output_scale_f=output_scale, output_add_scale_f=output_add_scale, outputs=1, @@ -444,10 +473,9 @@ def forward( mask_argsort_fwd_splits = [mask_argsort_fwd_splits] assert fp32_accum is None, "fp32_accum is not supported" - assert bias is None, "bias is not supported" assert scale is None assert output_add is None - assert output_dtype is torch.float32 + assert output_dtype == torch.float32, f"expected float32 output dtype, got {output_dtype!r}" # NOTE(knzo25): end of custom changes needed for deployment @@ -463,13 +491,22 @@ def forward( if output_dtype is None: output_dtype = features.dtype - alloc = TorchAllocator(features.device, features.dtype == torch.qint8) + is_features_qint8 = features.dtype == torch.qint8 + alloc = TorchAllocator(features.device, is_features_qint8) features_tv = torch_tensor_to_tv(features) pair_fwd_tv = torch_tensor_to_tv(pair_fwd) pair_mask_fwd_splits_tv = [torch_tensor_to_tv(t, tv.uint32) for t in pair_mask_fwd_splits] mask_argsort_fwd_splits_tv = [torch_tensor_to_tv(t) for t in mask_argsort_fwd_splits] filters_tv = torch_tensor_to_tv(filters) + if bias is not None: + if not bias.is_contiguous(): + bias = bias.contiguous() + assert bias.dim() == 1, f"bias must be 1D [C_out], got shape={tuple(bias.shape)}" + assert ( + bias.shape[0] == filters.shape[0] + ), f"bias shape mismatch: bias.shape[0]={bias.shape[0]} vs C_out={filters.shape[0]}" + bias_tv = torch_tensor_to_tv(bias) mask = np.array([np.iinfo(np.uint32).max], dtype=np.uint32) mask_tv = tv.from_numpy(mask).clone() timer_cpp = tv.CUDAKernelTimer(False) From 14902c4c5cdd356b16f173b1eef7bb52cd81559d Mon Sep 17 00:00:00 2001 From: vividf Date: Mon, 13 Jul 2026 15:06:26 +0900 Subject: [PATCH 4/4] feat: refactor BEVFusion deployment framework and add explanation docs Unify BEVFusion and CenterPoint under a shared deployment framework: restructure the export/inference/evaluation/verification layers, remove profiling and unused per-model deploy configs, rename the BEVFusion project to bevfusion_l, and add README/architecture docs explaining the BEVFusion deployment pipeline. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/settings.json | 7 + .gitignore | 4 + deployment/README.md | 11 +- deployment/cli/main.py | 9 +- deployment/config/base.py | 13 - deployment/config/enums.py | 91 +- deployment/config/schema.py | 10 + deployment/docs/REFACTOR_PLAN.md | 95 + deployment/docs/architecture.md | 23 +- deployment/docs/contributing.md | 22 +- deployment/docs/runbook.md | 6 - deployment/evaluation/base_evaluator.py | 10 +- ...evaluator.py => detection_3d_evaluator.py} | 112 +- .../evaluation/point_detection_executor.py | 101 - deployment/execution/__init__.py | 6 + .../backend_executor.py | 6 +- .../execution/point_cloud_backend_executor.py | 46 + deployment/export/contexts.py | 27 - .../export/exporters/tensorrt_exporter.py | 2 +- .../export/pipelines/component_builder.py | 11 +- .../export/pipelines/onnx_export_pipeline.py | 47 +- .../inference/base_inference_pipeline.py | 78 +- deployment/inference/gpu_resource_mixin.py | 12 + deployment/inference/tensorrt_runner.py | 178 ++ deployment/io/mmdet3d_model.py | 50 + deployment/io/point_cloud_data_loader.py | 93 + deployment/primitives/artifacts.py | 4 - .../evaluator_types.py | 0 deployment/primitives/tensorrt_plugins.py | 33 +- deployment/projects/bevfusion/cli.py | 16 - .../bevfusion/config/deploy_config.py | 163 -- deployment/projects/bevfusion/entrypoint.py | 174 -- .../bevfusion/evaluation/evaluator.py | 107 - .../projects/bevfusion/evaluation/executor.py | 58 - .../bevfusion/export/onnx_export_pipeline.py | 732 ----- .../onnx_fuse_implicit_gemm_activation.py | 148 - .../export/sparse_encoder_float_shadow.py | 295 -- .../export/tensorrt_export_pipeline.py | 136 - .../inference/bevfusion_inference_pipeline.py | 310 -- .../inference/onnx_inference_pipeline.py | 165 -- .../inference/pytorch_inference_pipeline.py | 234 -- .../inference/tensorrt_inference_pipeline.py | 808 ------ .../bevfusion/inference/trt_profiling.py | 188 -- .../projects/bevfusion/io/component_utils.py | 100 - .../projects/bevfusion/io/coors_contract.py | 29 - .../projects/bevfusion/io/data_loader.py | 82 - .../projects/bevfusion/io/model_loader.py | 65 - deployment/projects/bevfusion/runner.py | 92 - .../{bevfusion => bevfusion_l}/README.md | 26 +- .../{bevfusion => bevfusion_l}/__init__.py | 6 +- .../config/bevfusion_deployment_config.py | 64 + .../bevfusion_l/config/component_layout.py | 92 + .../bevfusion_l/config/deploy_config.py | 220 ++ .../config/deploy_config_original.py | 198 ++ .../config/deploy_config_without_opt.py} | 50 +- ...OWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md | 16 +- ...ME_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md | 12 +- .../28_README_BEVFUSION_2_8_DEPLOYMENT.md | 10 +- .../29_README_ONNX_NODE_COUNT_ALIGNMENT.md | 312 ++ ..._README_EVALUATION_PIPELINE_WALKTHROUGH.md | 588 ++++ ...ODEL_ARCHITECTURE_AND_SHAPE_WALKTHROUGH.md | 494 ++++ .../32_README_MODEL_ARCHITECTURE_detailed.md | 2540 +++++++++++++++++ ..._README_MODEL_ARCHITECTURE_code_mapping.md | 392 +++ .../{bevfusion => bevfusion_l}/docs/README.md | 6 +- deployment/projects/bevfusion_l/entrypoint.py | 31 + .../evaluation/__init__.py | 0 .../bevfusion_l/evaluation/executor.py | 97 + .../export/__init__.py | 0 .../bevfusion_l/export/component_builder.py | 138 + .../onnx_fuse_implicit_gemm_activation.py | 143 + .../export/onnx_models}/__init__.py | 0 .../export/onnx_models/bevfusion_onnx.py | 83 + .../bevfusion_l/export/sample_extractor.py | 54 + .../export/spconv_bn_fusion.py | 0 .../projects/bevfusion_l/export/transforms.py | 197 ++ .../io => bevfusion_l/inference}/__init__.py | 0 .../inference/bevfusion_inference_pipeline.py | 284 ++ .../inference/pytorch_inference_pipeline.py | 87 + .../inference/tensorrt_inference_pipeline.py | 230 ++ .../projects/bevfusion_l/io/__init__.py | 0 .../projects/bevfusion_l/io/head_outputs.py | 35 + .../projects/bevfusion_l/io/model_loader.py | 82 + .../projects/bevfusion_l/io/sample_types.py | 31 + .../projects/bevfusion_l/io/voxel_inputs.py | 72 + deployment/projects/bevfusion_l/runner.py | 103 + deployment/projects/centerpoint/README.md | 12 +- deployment/projects/centerpoint/__init__.py | 5 +- deployment/projects/centerpoint/cli.py | 14 - .../config/centerpoint_deployment_config.py | 41 + .../centerpoint/config/deploy_config.py | 9 +- .../deploy_config_fp16_convnext_small.py | 166 -- .../deploy_config_fp16_convnext_standard.py | 162 -- .../config/deploy_config_fp16_resnet.py | 162 -- .../config/deploy_config_fp16_resnet_base.py | 162 -- .../config/deploy_config_fp16_second.py | 162 -- .../config/deploy_config_fp16_second_2_5.py | 162 -- .../config/deploy_config_fp16_second_2_6.py | 160 -- .../config/deploy_config_fp16_second_base.py | 162 -- .../config/deploy_config_fp16_vov57.py | 162 -- .../config/deploy_config_fp16_vov99.py | 162 -- .../centerpoint/config/deploy_config_fp32.py | 162 -- deployment/projects/centerpoint/contexts.py | 21 - deployment/projects/centerpoint/entrypoint.py | 71 +- .../centerpoint/evaluation/evaluator.py | 53 - .../centerpoint/evaluation/executor.py | 76 +- .../centerpoint/export/component_builder.py | 25 +- .../centerpoint_inference_pipeline.py | 111 +- .../inference/onnx_inference_pipeline.py | 2 +- .../inference/tensorrt_inference_pipeline.py | 155 +- .../projects/centerpoint/io/data_loader.py | 141 - .../projects/centerpoint/io/model_loader.py | 56 +- .../projects/centerpoint/io/sample_types.py | 36 +- deployment/projects/centerpoint/runner.py | 41 +- deployment/projects/registry.py | 39 +- deployment/runtime/detection3d_entrypoint.py | 91 + deployment/runtime/evaluation_orchestrator.py | 2 +- deployment/runtime/export_orchestrator.py | 17 +- deployment/runtime/runner.py | 16 +- .../runtime/verification_orchestrator.py | 51 +- deployment/tests/test_centerpoint_configs.py | 8 +- deployment/tests/test_export_orchestrator.py | 9 +- deployment/tests/test_output_comparator.py | 2 +- deployment/verification/__init__.py | 6 + .../backend_verifier.py | 64 +- .../output_comparator.py | 10 +- deployment/verification/reporting.py | 20 + .../BEVFusion/bevfusion/bevfusion_head.py | 66 +- projects/BEVFusion/bevfusion/utils.py | 78 + projects/CenterPoint/README.md | 7 +- 129 files changed, 7961 insertions(+), 6880 deletions(-) create mode 100644 .claude/settings.json create mode 100644 deployment/docs/REFACTOR_PLAN.md rename deployment/evaluation/{detection3d_evaluator.py => detection_3d_evaluator.py} (50%) delete mode 100644 deployment/evaluation/point_detection_executor.py create mode 100644 deployment/execution/__init__.py rename deployment/{evaluation => execution}/backend_executor.py (96%) create mode 100644 deployment/execution/point_cloud_backend_executor.py delete mode 100644 deployment/export/contexts.py create mode 100644 deployment/inference/tensorrt_runner.py create mode 100644 deployment/io/mmdet3d_model.py create mode 100644 deployment/io/point_cloud_data_loader.py rename deployment/{evaluation => primitives}/evaluator_types.py (100%) delete mode 100644 deployment/projects/bevfusion/cli.py delete mode 100644 deployment/projects/bevfusion/config/deploy_config.py delete mode 100644 deployment/projects/bevfusion/entrypoint.py delete mode 100644 deployment/projects/bevfusion/evaluation/evaluator.py delete mode 100644 deployment/projects/bevfusion/evaluation/executor.py delete mode 100644 deployment/projects/bevfusion/export/onnx_export_pipeline.py delete mode 100644 deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py delete mode 100644 deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py delete mode 100644 deployment/projects/bevfusion/export/tensorrt_export_pipeline.py delete mode 100644 deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py delete mode 100644 deployment/projects/bevfusion/inference/onnx_inference_pipeline.py delete mode 100644 deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py delete mode 100644 deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py delete mode 100644 deployment/projects/bevfusion/inference/trt_profiling.py delete mode 100644 deployment/projects/bevfusion/io/component_utils.py delete mode 100644 deployment/projects/bevfusion/io/coors_contract.py delete mode 100644 deployment/projects/bevfusion/io/data_loader.py delete mode 100644 deployment/projects/bevfusion/io/model_loader.py delete mode 100644 deployment/projects/bevfusion/runner.py rename deployment/projects/{bevfusion => bevfusion_l}/README.md (73%) rename deployment/projects/{bevfusion => bevfusion_l}/__init__.py (70%) create mode 100644 deployment/projects/bevfusion_l/config/bevfusion_deployment_config.py create mode 100644 deployment/projects/bevfusion_l/config/component_layout.py create mode 100644 deployment/projects/bevfusion_l/config/deploy_config.py create mode 100644 deployment/projects/bevfusion_l/config/deploy_config_original.py rename deployment/projects/{bevfusion/config/deploy_config_split_fp16_opt_2_8.py => bevfusion_l/config/deploy_config_without_opt.py} (77%) rename deployment/projects/{bevfusion => bevfusion_l}/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md (92%) rename deployment/projects/{bevfusion => bevfusion_l}/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md (94%) rename deployment/projects/{bevfusion => bevfusion_l}/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md (95%) create mode 100644 deployment/projects/bevfusion_l/docs/29_README_ONNX_NODE_COUNT_ALIGNMENT.md create mode 100644 deployment/projects/bevfusion_l/docs/30_README_EVALUATION_PIPELINE_WALKTHROUGH.md create mode 100644 deployment/projects/bevfusion_l/docs/31_README_MODEL_ARCHITECTURE_AND_SHAPE_WALKTHROUGH.md create mode 100644 deployment/projects/bevfusion_l/docs/32_README_MODEL_ARCHITECTURE_detailed.md create mode 100644 deployment/projects/bevfusion_l/docs/33_README_MODEL_ARCHITECTURE_code_mapping.md rename deployment/projects/{bevfusion => bevfusion_l}/docs/README.md (54%) create mode 100644 deployment/projects/bevfusion_l/entrypoint.py rename deployment/projects/{bevfusion => bevfusion_l}/evaluation/__init__.py (100%) create mode 100644 deployment/projects/bevfusion_l/evaluation/executor.py rename deployment/projects/{bevfusion => bevfusion_l}/export/__init__.py (100%) create mode 100644 deployment/projects/bevfusion_l/export/component_builder.py create mode 100644 deployment/projects/bevfusion_l/export/onnx_fuse_implicit_gemm_activation.py rename deployment/projects/{bevfusion/inference => bevfusion_l/export/onnx_models}/__init__.py (100%) create mode 100644 deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py create mode 100644 deployment/projects/bevfusion_l/export/sample_extractor.py rename deployment/projects/{bevfusion => bevfusion_l}/export/spconv_bn_fusion.py (100%) create mode 100644 deployment/projects/bevfusion_l/export/transforms.py rename deployment/projects/{bevfusion/io => bevfusion_l/inference}/__init__.py (100%) create mode 100644 deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py create mode 100644 deployment/projects/bevfusion_l/inference/pytorch_inference_pipeline.py create mode 100644 deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py create mode 100644 deployment/projects/bevfusion_l/io/__init__.py create mode 100644 deployment/projects/bevfusion_l/io/head_outputs.py create mode 100644 deployment/projects/bevfusion_l/io/model_loader.py create mode 100644 deployment/projects/bevfusion_l/io/sample_types.py create mode 100644 deployment/projects/bevfusion_l/io/voxel_inputs.py create mode 100644 deployment/projects/bevfusion_l/runner.py delete mode 100644 deployment/projects/centerpoint/cli.py create mode 100644 deployment/projects/centerpoint/config/centerpoint_deployment_config.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py delete mode 100644 deployment/projects/centerpoint/config/deploy_config_fp32.py delete mode 100644 deployment/projects/centerpoint/contexts.py delete mode 100644 deployment/projects/centerpoint/evaluation/evaluator.py delete mode 100644 deployment/projects/centerpoint/io/data_loader.py create mode 100644 deployment/runtime/detection3d_entrypoint.py create mode 100644 deployment/verification/__init__.py rename deployment/{evaluation => verification}/backend_verifier.py (87%) rename deployment/{evaluation => verification}/output_comparator.py (96%) create mode 100644 deployment/verification/reporting.py diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..090799c0f --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(python3 -c ' *)" + ] + } +} diff --git a/.gitignore b/.gitignore index febfd80c6..3044c0685 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ # autoware-ml /data /work_dirs +/deployment/graphify-out +/graphify-out +.gitignore +CLAUDE.md *.bin *.onnx diff --git a/deployment/README.md b/deployment/README.md index 835bffbad..963733e16 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -16,8 +16,7 @@ python -m deployment.cli.main [--l # Example (CenterPoint) python -m deployment.cli.main centerpoint \ deployment/projects/centerpoint/config/deploy_config.py \ - \ - --rot-y-axis-reference + ``` ## What to read @@ -46,12 +45,14 @@ deployment/ ├── cli/ # Unified CLI ├── config/ # Typed deploy config schema ├── io/ # Data-loader base and sample types -├── export/ # exporters/ (ONNX/TensorRT), pipelines/, ExportContext +├── export/ # exporters/ (ONNX/TensorRT), pipelines/ ├── inference/ # Shared inference pipeline base and GPU resource helpers -├── evaluation/ # Evaluator, backend executor, verifier, output comparison +├── execution/ # BackendExecutor primitives (shared by evaluation + verification) +├── evaluation/ # Evaluators and metrics scoring +├── verification/ # Cross-backend numerical comparison (BackendVerifier, OutputComparator) ├── metrics/ # Task metrics interfaces (3D/2D detection, classification) ├── runtime/ # BaseDeploymentRunner, orchestrators, ArtifactManager -├── primitives/ # Cross-cutting leaf types: device (DeviceSpec), artifacts (path resolution) +├── primitives/ # Cross-cutting leaf types: device (DeviceSpec), artifacts, evaluator_types ├── projects/ # Per-task bundles └── tests/ # CPU-only unit tests (pytest) ``` diff --git a/deployment/cli/main.py b/deployment/cli/main.py index 2c7bbb292..1dae04775 100644 --- a/deployment/cli/main.py +++ b/deployment/cli/main.py @@ -2,7 +2,7 @@ Single deployment entrypoint. Usage: - python -m deployment.cli.main [project-specific args] + python -m deployment.cli.main [--log-level LEVEL] """ from __future__ import annotations @@ -51,7 +51,7 @@ def build_parser() -> argparse.ArgumentParser: subparsers = parser.add_subparsers(dest="project", required=True) - # Discover projects and import them so they can contribute args. + # Discover and import project packages so they register their adapters. failed_projects: list[str] = [] for project_name in _discover_project_packages(): try: @@ -61,14 +61,15 @@ def build_parser() -> argparse.ArgumentParser: failed_projects.append(f"- {project_name}: {e}\n{tb}") continue + # Only expose a subparser for projects that actually registered an adapter + # (get() raises KeyError if import ran but registration did not happen). try: - adapter = project_registry.get(project_name) + project_registry.get(project_name) except KeyError: continue sub = subparsers.add_parser(project_name, help=f"{project_name} deployment") parse_base_args(sub) # adds deploy_cfg, model_cfg, --log-level - adapter.add_args(sub) sub.set_defaults(_adapter_name=project_name) if not project_registry.list_projects(): diff --git a/deployment/config/base.py b/deployment/config/base.py index 886e5142e..70e807eff 100644 --- a/deployment/config/base.py +++ b/deployment/config/base.py @@ -51,8 +51,6 @@ def __init__(self, deploy_cfg: Config) -> None: Args: deploy_cfg: MMEngine Config object containing deployment settings """ - self._deploy_cfg = deploy_cfg - checkpoint_path = deploy_cfg.get("checkpoint_path") self.checkpoint_path = self._validate_checkpoint_path(checkpoint_path) self.device_config = DeviceConfig.from_dict(deploy_cfg.get("devices", {})) @@ -143,17 +141,6 @@ def resolved_deploy_log_file(self) -> Optional[str]: work_dir = Path(self.export_config.work_dir).expanduser() return str((work_dir / log_path).resolve(strict=False)) - @property - def deploy_cfg(self) -> Config: - """Raw deploy-config object (read-only). - - Surfaces the original MMEngine ``Config`` so project-specific export pipelines can read - project-only keys (e.g. BEVFusion's ``fuse_spconv_bn``, ``bevfusion_merge``, - ``spconv_do_sort``, ``spconv_fuse_implicit_gemm_relu``) that the typed sections - intentionally do not model. - """ - return self._deploy_cfg - def get_verification_scenarios(self, export_mode: ExportMode) -> Tuple[VerificationScenario, ...]: """ Get verification scenarios for the given export mode. diff --git a/deployment/config/enums.py b/deployment/config/enums.py index eb3470743..21961d56e 100644 --- a/deployment/config/enums.py +++ b/deployment/config/enums.py @@ -7,11 +7,47 @@ from __future__ import annotations from enum import Enum -from typing import Optional, Union +from typing import Optional, Type, TypeVar, Union # Constants DEFAULT_WORKSPACE_SIZE = 1 << 30 # 1 GB +_E = TypeVar("_E", bound=Enum) + + +def _enum_from_value( + enum_cls: Type[_E], + value: object, + *, + default: Optional[_E] = None, + label: Optional[str] = None, +) -> _E: + """Normalize a string or enum member into ``enum_cls`` (shared by the config enums). + + Matching is case-insensitive on the member ``value``. ``None`` returns ``default`` when + one is given (for optional config sections) and is otherwise an error, so every enum + parses identically instead of each hand-rolling its own ``from_value``. + + Raises: + ValueError: If ``value`` is ``None`` without a default, or is an unknown string. + TypeError: If ``value`` is neither ``None``, a ``str``, nor an ``enum_cls`` member. + """ + label = label or enum_cls.__name__ + valid = [member.value for member in enum_cls] + if value is None: + if default is not None: + return default + raise ValueError(f"{label} is required; must be one of {valid}.") + if isinstance(value, enum_cls): + return value + if isinstance(value, str): + normalized = value.strip().lower() + for member in enum_cls: + if member.value == normalized: + return member + raise ValueError(f"Invalid {label} '{value}'. Must be one of {valid}.") + raise TypeError(f"{label} must be a string or {enum_cls.__name__}, got {type(value).__name__}.") + class PrecisionPolicy(str, Enum): """Precision policy options for TensorRT. @@ -29,16 +65,10 @@ class PrecisionPolicy(str, Enum): @classmethod def from_value(cls, value: Optional[Union[str, PrecisionPolicy]]) -> PrecisionPolicy: """Parse strings or enum members into PrecisionPolicy (defaults to AUTO).""" - if value is None: - return cls.AUTO - if isinstance(value, cls): - return value - if isinstance(value, str): - normalized = value.strip().lower() - for member in cls: - if member.value == normalized: - return member - raise ValueError(f"Invalid precision_policy '{value}'. Must be one of {[m.value for m in cls]}.") + return _enum_from_value(cls, value, default=cls.AUTO, label="precision_policy") + + def __str__(self) -> str: # pragma: no cover - convenience for logging + return self.value class Backend(str, Enum): @@ -50,29 +80,8 @@ class Backend(str, Enum): @classmethod def from_value(cls, value: Union[str, Backend]) -> Backend: - """ - Normalize backend identifiers coming from configs or enums. - - Args: - value: Backend as string or Backend enum - - Returns: - Backend enum instance - - Raises: - ValueError: If value cannot be mapped to a supported backend - """ - if isinstance(value, cls): - return value - - if isinstance(value, str): - normalized = value.strip().lower() - try: - return cls(normalized) - except ValueError as exc: - raise ValueError(f"Unsupported backend '{value}'. Expected one of {[b.value for b in cls]}.") from exc - - raise TypeError(f"Backend must be a string or Backend enum, got {type(value)}") + """Normalize a backend identifier (string or enum) into a ``Backend`` member.""" + return _enum_from_value(cls, value, label="backend") @property def requires_cuda(self) -> bool: @@ -97,13 +106,7 @@ class ExportMode(str, Enum): @classmethod def from_value(cls, value: Optional[Union[str, ExportMode]]) -> ExportMode: """Parse strings or enum members into ExportMode (defaults to BOTH).""" - if value is None: - return cls.BOTH - if isinstance(value, cls): - return value - if isinstance(value, str): - normalized = value.strip().lower() - for member in cls: - if member.value == normalized: - return member - raise ValueError(f"Invalid export mode '{value}'. Must be one of {[m.value for m in cls]}.") + return _enum_from_value(cls, value, default=cls.BOTH, label="export mode") + + def __str__(self) -> str: # pragma: no cover - convenience for logging + return self.value diff --git a/deployment/config/schema.py b/deployment/config/schema.py index 22401ae40..64ab3b4ad 100644 --- a/deployment/config/schema.py +++ b/deployment/config/schema.py @@ -237,6 +237,16 @@ def items(self) -> Iterable[Tuple[str, ComponentCfg]]: """Iterate (name, ComponentCfg) pairs.""" return self._components.items() + def with_component(self, component: ComponentCfg) -> ComponentsConfig: + """Return a new ``ComponentsConfig`` with ``component`` added (replacing any of the same name). + + Lets callers derive a layout (e.g. BEVFusion's merged ``bevfusion_merged``) from already + typed components without round-tripping the whole config back through raw dicts. + """ + return ComponentsConfig( + _components=MappingProxyType({**self._components, component.name: component}), + ) + @staticmethod def _validate_dynamic_axes(raw: Any) -> Dict[str, Dict[int, str]]: """Validate dynamic_axes schema without coercing types.""" diff --git a/deployment/docs/REFACTOR_PLAN.md b/deployment/docs/REFACTOR_PLAN.md new file mode 100644 index 000000000..aff97461b --- /dev/null +++ b/deployment/docs/REFACTOR_PLAN.md @@ -0,0 +1,95 @@ +# Deployment Framework Cleanup & Refactor Plan + +Working checklist for the cleanliness pass across the shared framework and the +`bevfusion_l` / `centerpoint` project bundles. Goal: clear responsibilities, clear +naming, high readability, no smells / hard-code / over-engineering, and BEVFusion aligned +with CenterPoint's clean architecture (fixing the shared layer once where both diverge). + +Status legend: `[ ]` todo · `[x]` done · `[~]` intentionally skipped / deferred (documented). + +Verification note: the host has no torch/CUDA/mmengine (see project memory), so each change is +verified statically — `ast.parse`, `pyflakes`, targeted `grep`, CLI package discovery, and +`exec()` of the (pure-Python) deploy config. A Docker e2e smoke test (`bevfusion_l` / `centerpoint` +export+eval) is still recommended after the pass and is **not** covered here. + +--- + +## Tier A — safe fixes (dead code / one real bug) + +- [x] **A-BUG** `OutputComparator._merge_summaries` `inf * 0 = nan` fixed by skipping zero-element + children in the weighted mean. Verified: mixed shape-mismatch + valid child now yields finite + `mean_diff` with `max_diff=inf`. `verification/output_comparator.py`. +- [x] **A1** `VerificationOrchestrator` now logs `verification_results["error"]` (device-validation + failures) and `continue`s instead of counting the scenario as 0/0. +- [x] **A2** Removed dead `TensorDiffDetail.passed` field + both construction sites. +- [x] **A3** Removed unused module `logger` (and the `logging` import) from `primitives/artifacts.py`. +- [x] **A4** `tensorrt_plugins.py`: dropped dead `loaded_now`, collapsed the duplicated `CDLL` + branch to one call + conditional log, and now returns the libraries newly loaded by this call + (matches the docstring) instead of the cumulative global set. + +## Tier B — shared consistency (fix once, helps both projects) + +- [x] **A5** Added `_enum_from_value` helper in `config/enums.py`; `PrecisionPolicy` / `ExportMode` + / `Backend` all route through it (consistent None-default + `ValueError`/`TypeError`). Behavior + smoke-tested. +- [x] **A7** Added `verification/reporting.py` (`BANNER_WIDTH=80`, `banner()`, `format_verdict()`); + applied in `backend_verifier.py` (was `60` + inline emoji) and `verification_orchestrator.py` + (was `80`). Also unified the counter noun to "samples" and the `policy`→`scenario` terminology. +- [x] **A8** `load_tensorrt_plugin_libraries` no longer takes an injected `logger` (uses a module + logger). Both call sites updated: `bevfusion_l` TRT pipeline **and** the shared + `export/exporters/tensorrt_exporter.py` (the second caller — caught by the grep sweep). +- [x] **A9** Consistent `__str__` on `PrecisionPolicy` / `ExportMode` / `Backend` (all return `.value`). +- [x] **(bonus)** Fixed the garbled `_fmt_finite_diff` docstring in `backend_verifier.py`. + +## Tier C — BEVFusion cleanup (align with CenterPoint) + +- [x] **P0-2** BEVFusion TRT pipeline now stores `self._engines` / `self._contexts` dicts keyed by + component name (CenterPoint pattern). 6 attributes + `_split` branching → 2 dicts + a uniform + `_load_tensorrt_engines` loop and a single-line `_release_gpu_resources`. +- [x] **P1-2** Runner docstring trimmed from ~20 lines to CenterPoint brevity. +- [x] **P1-3** Dropped `_pick_bound_input_name` + its "mAP=0" warning; the single-input dense engine + now binds `input_names[0]`. `strict=False` output ordering **kept deliberately** (BEVFusion export + had name drift; aligning to CenterPoint's `strict=True` is unsafe without an e2e run). + +## Tier D — config cleanliness + +- [x] **P1-1** `deploy_config.py` restructured to CenterPoint's numbered-section layout with hoisted + single-source `_` literals (`_CUDA`, `_WORK_DIR`/`_ONNX_DIR`/`_TENSORRT_DIR`, `_LIDAR_BEV_SHAPE`, + voxel-profile literals) and a cleaned/accurate docstring. **All values preserved** — verified by + `exec()`-ing the file and asserting every resolved value (incl. `engine_dir`) matches the original. +- [~] **P0-3 (DEFERRED)** `_base_` dedup of the two variants. The base bakes computed paths + (`_TENSORRT_DIR` → `evaluation.backends.tensorrt.engine_dir`), so a child overriding + `export.work_dir` would silently keep the base's `engine_dir` unless it also overrides that nested + path — fragile, and unverifiable here (no mmengine to run `Config.fromfile`). Do this in Docker + where the resolved dicts can be asserted equal, or after refactoring the base to not bake derived + paths. Left the three explicit configs as-is (they work). + +## Tier E — cross-project entrypoint dedup + +- [x] **P0-1** Added `runtime/detection3d_entrypoint.py::run_detection3d_deployment(...)`. Both + `bevfusion_l/entrypoint.py` and `centerpoint/entrypoint.py` are now ~30 lines that inject only + `pipeline_name` + `config_factory` + `executor_factory` + `runner_factory`. The ~90% duplicated + wiring lives once. + +--- + +## Intentionally skipped (cosmetic / would be churn, not value) + +- [~] `DeviceSpec.to_ort_provider` / `to_torch_device` "leaky primitive" — pragmatic, low ROI. +- [~] 44-line docstring on `resolve_artifact_path`; `# ===` section banners; PEP585-vs-`typing` + generics mixing; `__post_init__` re-validating a `Literal`; `Artifact.exists` (dir-OK) vs + resolver `is_file()`. +- [~] model_loader strategy divergence (CenterPoint type-swap vs BEVFusion wrappers), split/merged + branching, CPU-vs-CUDA load, 2-vs-3 backends, component naming — justified divergences. + +--- + +## Re-audit results + +- [x] `ast.parse` clean on all 17 changed `.py`. +- [x] `pyflakes` clean (no unused imports / undefined names) on all changed `.py`. +- [x] CLI package discovery lists exactly `['bevfusion_l', 'centerpoint']`. +- [x] grep: no references to removed symbols (`_pick_bound_input_name`, `_engine_sparse`, + `_apply_spconv_do_sort`, `_get_num_proposals`, 2-arg `load_tensorrt_plugin_libraries`). +- [x] enum + comparator-nan behavior smoke-tested; deploy_config values `exec()`-verified. +- [ ] **Docker e2e smoke test** (`bevfusion_l` + `centerpoint` export/eval) — REQUIRED, not run here. diff --git a/deployment/docs/architecture.md b/deployment/docs/architecture.md index 9440caeaa..414b3f678 100644 --- a/deployment/docs/architecture.md +++ b/deployment/docs/architecture.md @@ -54,12 +54,14 @@ flowchart TD | `deployment/cli/` | Unified CLI and shared argument helpers | | `deployment/config/` | Typed deployment config and schema | | `deployment/io/` | Data-loader base and sample types | -| `deployment/export/` | ONNX/TensorRT exporters (`exporters/`), export pipeline bases (`pipelines/`), and the `ExportContext` base | +| `deployment/export/` | ONNX/TensorRT exporters (`exporters/`) and export pipeline bases (`pipelines/`) | | `deployment/inference/` | Shared inference pipeline base and GPU resource helpers | -| `deployment/evaluation/` | Base evaluator, backend executor, backend verifier, and output-comparison helpers | +| `deployment/execution/` | `BackendExecutor` primitives (backend pipeline creation, input prep, device handling) — the shared stage that evaluation and verification both build on | +| `deployment/evaluation/` | Base evaluator and task evaluators (`Detection3DEvaluator`) — metrics scoring | +| `deployment/verification/` | Cross-backend numerical comparison (`BackendVerifier`, `OutputComparator`) — a peer stage to evaluation, not a form of it | | `deployment/metrics/` | Task metrics interfaces (3D/2D detection, classification) | | `deployment/runtime/` | Base runner, orchestrators, and artifact management | -| `deployment/primitives/` | Cross-cutting leaf types used by every stage: `device` (`DeviceSpec`) and `artifacts` (path resolution) — not a pipeline stage | +| `deployment/primitives/` | Cross-cutting leaf types used by every stage: `device` (`DeviceSpec`), `artifacts` (path resolution), and `evaluator_types` (shared value types: `InferenceInput`, `ModelSpec`, result dicts) — not a pipeline stage | | `deployment/projects//` | Project-specific entrypoint, runner, config, io, export, inference, evaluation, and optional contexts logic (see the project layout contract below) | ## Extension contract @@ -111,16 +113,14 @@ implement. When you author a new project, walk this table top to bottom — ever | Project path (`projects//`) | Mirrors framework module | What to implement | Required | | --- | --- | --- | --- | -| `__init__.py` | [`projects/registry.py`](../projects/registry.py) | Register a `ProjectAdapter` (`name`, `add_args`, `run`) | Required | +| `__init__.py` | [`projects/registry.py`](../projects/registry.py) | Register a `ProjectAdapter` (`name`, `run`) | Required | | `entrypoint.py` | [`cli/`](../cli) + `projects/registry.py` | A `run(args)` that builds config, loader, evaluator, runner | Required | -| `cli.py` | [`cli/args.py`](../cli/args.py) | `add_args(parser)` for project flags (may be a no-op) | Required | -| `config/` | [`config/`](../config) | Deploy config consumed as `BaseDeploymentConfig` | Required | +| `config/` | [`config/`](../config) | Deploy config consumed as `BaseDeploymentConfig` (subclass it to model project-specific keys as typed attributes) | Required | | `io/` | [`io/`](../io) | `BaseDataLoader` + `SampleData` subclasses | Required | | `inference/` | [`inference/`](../inference) | `BaseInferencePipeline` per backend (+ `GPUResourceMixin` for TensorRT) | Required | -| `evaluation/` | [`evaluation/`](../evaluation) | `BaseEvaluator` + `BackendExecutor` subclasses | Required | +| `evaluation/` | [`evaluation/`](../evaluation) + [`execution/`](../execution) | `evaluator.py` (a `BaseEvaluator`/`Detection3DEvaluator` subclass) + `executor.py` (a `BackendExecutor`/`PointCloudBackendExecutor` subclass); the executor base lives in `execution/`, its project subclass sits beside the evaluator here | Required | | `runner.py` | [`runtime/runner.py`](../runtime/runner.py) | Thin `BaseDeploymentRunner` subclass | Required | | `export/` | [`export/`](../export) | `ModelComponentBuilder` + `SampleExtractor` subclasses (in `export/pipelines/`); plus `export/onnx_models/` for export-time ONNX graph definitions | Optional | -| `contexts.py` | [`export/contexts.py`](../export/contexts.py) | `ExportContext` subclass — only if you need extra context fields | Optional | Metrics are intentionally **not** a project directory. Metrics configs, interfaces, and the extractors that build a config from a model config are @@ -134,9 +134,10 @@ introduce a genuinely new task, not a new model. Every stage directory mirrors its framework counterpart by the **same name** (`config`, `io`, `export`, `inference`, `evaluation`) — no exceptions. The only -non-mirrored items are the wiring single-files: `entrypoint.py`, `cli.py`, -`runner.py`, and `contexts.py`, which glue the project to `cli/`, -`runtime/runner.py`, and `export/contexts.py`. +non-mirrored items are the wiring single-files: `entrypoint.py` and `runner.py`, +which glue the project to `cli/` and `runtime/runner.py`. Projects have no +per-project CLI flags — everything that shapes the exported artifact lives in the +deploy config, so the CLI carries only `deploy_cfg`, `model_cfg`, and `--log-level`. ### Pipeline naming diff --git a/deployment/docs/contributing.md b/deployment/docs/contributing.md index 77df31f39..2155d5549 100644 --- a/deployment/docs/contributing.md +++ b/deployment/docs/contributing.md @@ -12,16 +12,18 @@ full project-path → framework-module → base-class mapping. 1. Create `deployment/projects//__init__.py` and register a `ProjectAdapter`. 2. Add `entrypoint.py` to build `BaseDeploymentConfig`, the data loader, evaluator, and runner. -3. Add `cli.py` exposing `add_args(parser)` for project flags (may be a no-op). -4. Add `runner.py` as a thin `BaseDeploymentRunner` subclass. -5. Add `configs/deploy_config.py` with the required deploy config sections described in [configuration.md](./configuration.md). -6. Add `io/` and `evaluation/` for project-specific loading and evaluation logic (mirroring framework `io/` and `evaluation/`). -7. In `entrypoint.py`, build the task metrics config from the shared `metrics/` (e.g. `extract_t4metric_v2_config` for 3D detection) and pass it to the evaluator — do **not** add a project `metrics/` directory. -8. Add `inference/` with backend-specific inference pipelines (files `*_inference_pipeline.py`), and create them from the project's `BackendExecutor.create_pipeline`. -9. Add a project `README.md` with the project-specific quick start and links back to shared docs. - -Add `contexts.py` only when the project needs extra `ExportContext` fields, and -`export/` only when the project needs multi-stage or multi-file export orchestration. +3. Add `runner.py` as a thin `BaseDeploymentRunner` subclass. +4. Add `configs/deploy_config.py` with the required deploy config sections described in [configuration.md](./configuration.md). +5. Add `io/` and `evaluation/` for project-specific loading and evaluation logic (mirroring framework `io/` and `evaluation/`). +6. In `entrypoint.py`, build the task metrics config from the shared `metrics/` (e.g. `extract_t4metric_v2_config` for 3D detection) and pass it to the evaluator — do **not** add a project `metrics/` directory. +7. Add `inference/` with backend-specific inference pipelines (files `*_inference_pipeline.py`), and create them from the project's `BackendExecutor.create_pipeline`. +8. Add a project `README.md` with the project-specific quick start and links back to shared docs. + +Projects have no per-project CLI flags: any option that shapes the exported artifact (e.g. +`rot_y_axis_reference`) belongs in the deploy config, modeled as a typed attribute on a +project-specific `BaseDeploymentConfig` subclass so it is versioned with the artifact. The CLI +carries only `deploy_cfg`, `model_cfg`, and `--log-level`. Add `export/` only when the project needs +multi-stage or multi-file export orchestration. ## Implementation notes diff --git a/deployment/docs/runbook.md b/deployment/docs/runbook.md index dcce40fb2..dd7bd7745 100644 --- a/deployment/docs/runbook.md +++ b/deployment/docs/runbook.md @@ -11,12 +11,6 @@ python -m deployment.cli.main centerpoint \ \ \ [--log-level INFO] - -# CenterPoint-specific flag -python -m deployment.cli.main centerpoint \ - \ - \ - --rot-y-axis-reference ``` ## Required inputs diff --git a/deployment/evaluation/base_evaluator.py b/deployment/evaluation/base_evaluator.py index 0d16332c4..4ed86f71c 100644 --- a/deployment/evaluation/base_evaluator.py +++ b/deployment/evaluation/base_evaluator.py @@ -22,16 +22,16 @@ import numpy as np from mmengine.config import Config -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.evaluation.evaluator_types import ( +from deployment.execution.backend_executor import BackendExecutor +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.io.base_data_loader import BaseDataLoader +from deployment.metrics.base_metrics_interface import BaseMetricsInterface +from deployment.primitives.evaluator_types import ( EvalResultDict, LatencyBreakdown, LatencyStats, ModelSpec, ) -from deployment.inference.base_inference_pipeline import BaseInferencePipeline -from deployment.io.base_data_loader import BaseDataLoader -from deployment.metrics.base_metrics_interface import BaseMetricsInterface logger = logging.getLogger(__name__) diff --git a/deployment/evaluation/detection3d_evaluator.py b/deployment/evaluation/detection_3d_evaluator.py similarity index 50% rename from deployment/evaluation/detection3d_evaluator.py rename to deployment/evaluation/detection_3d_evaluator.py index d168b4b19..04e915593 100644 --- a/deployment/evaluation/detection3d_evaluator.py +++ b/deployment/evaluation/detection_3d_evaluator.py @@ -1,45 +1,40 @@ -"""Shared evaluator for point-cloud 3D detectors (CenterPoint, BEVFusion). - -Factors out the metrics hooks (prediction/GT parsing, metric accumulation, result building, -comparison summary) that the two projects' evaluators previously duplicated (~6 methods, near -verbatim). Subclasses override only ``print_results`` (backend-specific latency-breakdown -layout) and may reuse ``_log_latency_stats`` for the shared latency block. - -Note: this unifies the two former ``_build_results`` copies to the **stricter** CenterPoint -behavior — it validates the required summary keys and populates ``detailed_metrics`` with the -computed metrics. The old BEVFusion copy was lax (``.get(..., {})`` and empty detailed metrics); -that drift is intentionally removed here. +"""Shared evaluator for 3D detectors. + +Scoring depends only on the 3D-detection outputs (``bbox_3d`` + ``label``), not on the input +modality, so any 3D detector — point-cloud (CenterPoint, BEVFusion) or camera-based — reuses this. +CenterPoint and BEVFusion score predictions the same way — via +:class:`~deployment.metrics.detection_3d_metrics.Detection3DMetricsInterface` — so the metrics +plumbing (prediction/GT parsing, result building, comparison summary) lives here once. Projects +subclass this and only override :meth:`print_results` when they want a custom latency layout; +the default here prints a generic metrics + latency + stage breakdown report. """ from __future__ import annotations import logging -from typing import Dict, List, Mapping +from typing import Any, Dict, List, Mapping import numpy as np from mmengine.config import Config from typing_extensions import override -from deployment.evaluation.backend_executor import BackendExecutor from deployment.evaluation.base_evaluator import BaseEvaluator, EvalResultDict +from deployment.execution.backend_executor import BackendExecutor from deployment.metrics.detection_3d_metrics import Detection3DMetricsConfig, Detection3DMetricsInterface logger = logging.getLogger(__name__) -_REQUIRED_SUMMARY_KEYS = ("mAP_by_mode", "mAPH_by_mode", "per_class_ap_by_mode") - class Detection3DEvaluator(BaseEvaluator): - """Evaluator base for 3D-detection deployment: metrics hooks shared across detectors. - - Backend execution (pipeline creation, input prep, device handling) is delegated to the - ``executor``; this class implements the task-generic metrics hooks. Subclasses provide only - ``print_results`` (the latency-breakdown layout differs per model). + """Evaluator for 3D detection backed by ``Detection3DMetricsInterface`` (modality-agnostic). Args: model_cfg: Model configuration; must have ``class_names``. metrics_config: Configuration for 3D detection metrics (e.g. T4MetricV2). executor: Backend execution primitives, shared with the verification runner. + + Raises: + ValueError: If ``model_cfg`` does not have ``class_names``. """ def __init__( @@ -50,6 +45,7 @@ def __init__( ) -> None: if not hasattr(model_cfg, "class_names"): raise ValueError("class_names must be provided via model_cfg.class_names.") + super().__init__( metrics_interface=Detection3DMetricsInterface(metrics_config), model_cfg=model_cfg, @@ -57,22 +53,30 @@ def __init__( ) @override - def _parse_predictions(self, pipeline_output: object) -> List[Dict]: + def _parse_predictions(self, pipeline_output: Any) -> List[Dict]: """Return pipeline output as a list of prediction dicts (empty list if not a list).""" return pipeline_output if isinstance(pipeline_output, list) else [] @override - def _parse_ground_truths(self, gt_data: Mapping[str, object]) -> List[Dict]: - """Convert ``gt_bboxes_3d`` / ``gt_labels_3d`` into ``[{"bbox_3d": [...], "label": int}]``.""" + def _parse_ground_truths(self, gt_data: Mapping[str, Any]) -> List[Dict]: + """Convert ``gt_bboxes_3d`` / ``gt_labels_3d`` into a list of ``{bbox_3d, label}`` dicts. + + Raises: + KeyError: If ``gt_bboxes_3d`` or ``gt_labels_3d`` is missing. + """ if "gt_bboxes_3d" not in gt_data: raise KeyError("gt_bboxes_3d not found in ground truth data.") if "gt_labels_3d" not in gt_data: raise KeyError("gt_labels_3d not found in ground truth data.") - gt_bboxes_3d = np.asarray(gt_data["gt_bboxes_3d"], dtype=np.float32) - box_dim = gt_bboxes_3d.shape[-1] if gt_bboxes_3d.ndim > 1 else 7 - gt_bboxes_3d = gt_bboxes_3d.reshape(-1, box_dim) - gt_labels_3d = np.asarray(gt_data["gt_labels_3d"], dtype=np.int64).reshape(-1) + gt_bboxes_3d = gt_data["gt_bboxes_3d"] + gt_labels_3d = gt_data["gt_labels_3d"] + + gt_bboxes_3d = np.asarray(gt_bboxes_3d, dtype=np.float32).reshape( + -1, np.asarray(gt_bboxes_3d).shape[-1] if np.asarray(gt_bboxes_3d).ndim > 1 else 7 + ) + gt_labels_3d = np.asarray(gt_labels_3d, dtype=np.int64).reshape(-1) + return [{"bbox_3d": gt_bboxes_3d[i].tolist(), "label": int(gt_labels_3d[i])} for i in range(len(gt_bboxes_3d))] @override @@ -87,11 +91,17 @@ def _build_results( latency_breakdowns: List[Dict[str, float]], num_samples: int, ) -> EvalResultDict: - """Build the result dict (mAP/mAPH, per-class AP, detailed metrics, latency, breakdown).""" + """Aggregate mAP/mAPH, per-class AP, latency, and optional breakdown into an EvalResultDict. + + Raises: + KeyError: If the metrics summary is missing required keys. + """ latency_stats = self.compute_latency_stats(latencies) + map_results = self.metrics_interface.compute_metrics() summary_dict = self.metrics_interface.summary.to_dict() - missing = [k for k in _REQUIRED_SUMMARY_KEYS if k not in summary_dict] + required_summary_keys = ("mAP_by_mode", "mAPH_by_mode", "per_class_ap_by_mode") + missing = [k for k in required_summary_keys if k not in summary_dict] if missing: raise KeyError(f"Missing required metrics summary keys: {missing}") @@ -103,8 +113,10 @@ def _build_results( "latency": latency_stats, "num_samples": num_samples, } + if latency_breakdowns: result["latency_breakdown"] = self._compute_latency_breakdown(latency_breakdowns) + return result @override @@ -118,8 +130,23 @@ def summarize_for_comparison(self, results: EvalResultDict) -> List[str]: lines.extend(super().summarize_for_comparison(results)) return lines + def _log_metrics_report(self) -> None: + """Log the metrics interface's formatted report line by line.""" + metrics_report = self.metrics_interface.format_metrics_report() + if metrics_report: + for line in metrics_report.rstrip().split("\n"): + logger.info(line) + def _log_latency_stats(self, results: EvalResultDict) -> None: - """Log the shared latency-statistics block (mean/std/min/max/median).""" + """Log the latency-statistics block (mean/std/min/max/median). + + Raises: + ValueError: If ``latency`` is missing from ``results``. + """ + if "latency" not in results: + raise ValueError( + "Latency statistics not found in results. Ensure that evaluation has been run with latency tracking." + ) latency_dict = results["latency"].to_dict() logger.info("") logger.info("Latency Statistics:") @@ -128,3 +155,28 @@ def _log_latency_stats(self, results: EvalResultDict) -> None: logger.info(" Min: %.2f ms", latency_dict["min_ms"]) logger.info(" Max: %.2f ms", latency_dict["max_ms"]) logger.info(" Median: %.2f ms", latency_dict["median_ms"]) + + @override + def print_results(self, results: EvalResultDict) -> None: + """Log the metrics report, latency statistics, and a generic stage-wise breakdown. + + Subclasses override this only when they want a custom breakdown layout. + """ + self._log_metrics_report() + self._log_latency_stats(results) + + if "latency_breakdown" in results: + breakdown_dict = results["latency_breakdown"].to_dict() + if breakdown_dict: + logger.info("") + logger.info("Stage-wise Latency Breakdown:") + top_level_stages = {"preprocessing_ms", "model_ms", "postprocessing_ms"} + for stage, stats_dict in breakdown_dict.items(): + stage_name = stage.replace("_ms", "").replace("_", " ").title() + output_format = ( + " %-18s: %.2f ± %.2f ms" if stage in top_level_stages else " %-16s: %.2f ± %.2f ms" + ) + logger.info(output_format, stage_name, stats_dict["mean_ms"], stats_dict["std_ms"]) + + logger.info("") + logger.info("Total Samples: %s", results["num_samples"]) diff --git a/deployment/evaluation/point_detection_executor.py b/deployment/evaluation/point_detection_executor.py deleted file mode 100644 index c1e32cd5b..000000000 --- a/deployment/evaluation/point_detection_executor.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Shared backend-execution primitives for point-cloud 3D detectors. - -``PointDetectionExecutor`` factors out the pipeline-construction and ``(points, metainfo)`` -input-prep that the CenterPoint and BEVFusion executors previously duplicated (~85% overlap). -Subclasses declare the three backend pipeline classes and, optionally, override -``get_output_names`` / ``_tensorrt_pipeline_kwargs``; everything else is shared. -""" - -from __future__ import annotations - -import logging -from typing import Any, Mapping, Optional, Type - -from typing_extensions import override - -from deployment.config.enums import Backend -from deployment.config.schema import ComponentsConfig -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.evaluation.evaluator_types import InferenceInput, ModelSpec -from deployment.inference.base_inference_pipeline import BaseInferencePipeline -from deployment.io.base_data_loader import BaseDataLoader -from deployment.primitives.device import DeviceSpec - -logger = logging.getLogger(__name__) - - -class PointDetectionExecutor(BackendExecutor): - """Backend execution primitives shared by point-cloud 3D detectors (CenterPoint, BEVFusion). - - Subclasses set the three pipeline classes and (optionally) override ``get_output_names`` - and ``_tensorrt_pipeline_kwargs``. Pipeline construction and input prep are shared. - - Args: - components_cfg: Unified components configuration, forwarded to the ONNX/TensorRT - pipelines so they can resolve split vs merged artifacts. - """ - - #: Human-readable task name, used in log lines and error messages. - task_name: str = "point detector" - #: Backend pipeline classes; set as class attributes by subclasses. - pytorch_pipeline_cls: Optional[Type[BaseInferencePipeline]] = None - onnx_pipeline_cls: Optional[Type[BaseInferencePipeline]] = None - tensorrt_pipeline_cls: Optional[Type[BaseInferencePipeline]] = None - - def __init__(self, components_cfg: ComponentsConfig) -> None: - super().__init__() - self._components_cfg = components_cfg - - def _tensorrt_pipeline_kwargs(self) -> Mapping[str, Any]: - """Extra keyword args forwarded to the TensorRT pipeline (default: none). - - BEVFusion overrides this to pass its custom spconv ImplicitGemm ``plugin_libraries``. - """ - return {} - - @override - def create_pipeline(self, model_spec: ModelSpec, device: DeviceSpec) -> BaseInferencePipeline: - """Create a backend inference pipeline for ``model_spec.backend`` on ``device``.""" - backend = model_spec.backend - self._validate_backend(backend) - - if backend is Backend.PYTORCH: - logger.info("Creating %s PyTorch pipeline on %s", self.task_name, device) - return self.pytorch_pipeline_cls(self.pytorch_model, device=device) - - if backend is Backend.ONNX: - logger.info("Creating %s ONNX pipeline from %s on %s", self.task_name, model_spec.artifact.path, device) - return self.onnx_pipeline_cls( - self.pytorch_model, - onnx_dir=model_spec.artifact.path, - device=device, - components_cfg=self._components_cfg, - ) - - if backend is Backend.TENSORRT: - logger.info( - "Creating %s TensorRT pipeline from %s on %s", self.task_name, model_spec.artifact.path, device - ) - return self.tensorrt_pipeline_cls( - self.pytorch_model, - tensorrt_dir=model_spec.artifact.path, - device=device, - components_cfg=self._components_cfg, - **self._tensorrt_pipeline_kwargs(), - ) - - raise ValueError(f"Unsupported backend: {backend.value}") - - @override - def prepare_input( - self, - sample: Mapping[str, Any], - data_loader: BaseDataLoader, - device: DeviceSpec, - ) -> InferenceInput: - """Build InferenceInput from a sample containing ``points`` and ``metainfo``.""" - if "points" not in sample: - raise ValueError(f"Expected 'points' in sample. Got keys: {list(sample.keys())}") - if "metainfo" not in sample: - raise KeyError(f"Sample must contain 'metainfo' for {self.task_name} postprocess.") - return InferenceInput(data=sample["points"], metadata=sample["metainfo"]) diff --git a/deployment/execution/__init__.py b/deployment/execution/__init__.py new file mode 100644 index 000000000..1963b83a2 --- /dev/null +++ b/deployment/execution/__init__.py @@ -0,0 +1,6 @@ +"""Backend execution primitives. Import concrete submodules (``deployment.execution.backend_executor``, …). + +Execution is the shared stage that both evaluation and verification build on: a ``BackendExecutor`` +turns a sample into an ``InferenceInput`` and runs it through a backend pipeline. It has no +dependency on the metrics (evaluation) or comparison (verification) layers. +""" diff --git a/deployment/evaluation/backend_executor.py b/deployment/execution/backend_executor.py similarity index 96% rename from deployment/evaluation/backend_executor.py rename to deployment/execution/backend_executor.py index 54681e67b..d52a249d6 100644 --- a/deployment/evaluation/backend_executor.py +++ b/deployment/execution/backend_executor.py @@ -7,7 +7,7 @@ It is shared by both `~deployment.evaluation.base_evaluator.BaseEvaluator` (the evaluation loop) and -`~deployment.evaluation.backend_verifier.BackendVerifier` (the +`~deployment.verification.backend_verifier.BackendVerifier` (the reference/test verification loop), so neither has to depend on the other. """ @@ -18,10 +18,10 @@ from typing import Any, List, Mapping, Optional from deployment.config.enums import Backend -from deployment.evaluation.evaluator_types import InferenceInput, ModelSpec from deployment.inference.base_inference_pipeline import BaseInferencePipeline from deployment.io.base_data_loader import BaseDataLoader from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import InferenceInput, ModelSpec logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def get_output_names(self) -> Optional[List[str]]: Override when the backend's pipeline returns a sequence of tensors with known semantic names (e.g. detection heads). The names are forwarded to the - `~deployment.evaluation.output_comparator.OutputComparator` to label + `~deployment.verification.output_comparator.OutputComparator` to label positions in diagnostic paths. Returns: diff --git a/deployment/execution/point_cloud_backend_executor.py b/deployment/execution/point_cloud_backend_executor.py new file mode 100644 index 000000000..b826bec14 --- /dev/null +++ b/deployment/execution/point_cloud_backend_executor.py @@ -0,0 +1,46 @@ +"""Shared backend-executor primitives for point-cloud 3D detectors. + +Point-cloud detectors (CenterPoint, BEVFusion, …) all feed the network the same per-sample +input: the raw ``points`` tensor plus the ``metainfo`` needed by postprocess. This base +implements that single shared ``prepare_input`` so each project executor only has to provide +the backend-specific ``create_pipeline`` (and optionally ``get_output_names``). +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from typing_extensions import override + +from deployment.execution.backend_executor import BackendExecutor +from deployment.io.base_data_loader import BaseDataLoader +from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import InferenceInput + + +class PointCloudBackendExecutor(BackendExecutor): + """BackendExecutor whose model input is ``(points, metainfo)`` (shared prepare_input).""" + + @override + def prepare_input( + self, + sample: Mapping[str, Any], + data_loader: BaseDataLoader, + device: DeviceSpec, + ) -> InferenceInput: + """Build InferenceInput from a sample's ``points`` + ``metainfo``. + + Args: + sample: Dict with 'points' and 'metainfo'. + data_loader: Unused; kept for interface compatibility. + device: Unused; kept for interface compatibility. + + Raises: + ValueError: If 'points' is missing from sample. + KeyError: If 'metainfo' is missing from sample. + """ + if "points" not in sample: + raise ValueError(f"Expected 'points' in sample. Got keys: {list(sample.keys())}") + if "metainfo" not in sample: + raise KeyError("Sample must contain 'metainfo' for postprocess.") + return InferenceInput(data=sample["points"], metadata=sample["metainfo"]) diff --git a/deployment/export/contexts.py b/deployment/export/contexts.py deleted file mode 100644 index 29bd21949..000000000 --- a/deployment/export/contexts.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Typed context objects for deployment workflows. - -Usage: - # Create context for export - ctx = ExportContext() - - # Pass to orchestrator - result = export_orchestrator.run(ctx) - -Project-specific subclasses live with their project (e.g. -``deployment.projects.centerpoint.contexts.CenterPointExportContext``). -""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ExportContext: - """ - Base context for export operations. - - Marker base class for export contexts; project-specific subclasses add - typed fields for their export parameters. - """ diff --git a/deployment/export/exporters/tensorrt_exporter.py b/deployment/export/exporters/tensorrt_exporter.py index 387b43e0b..35f60f8dd 100644 --- a/deployment/export/exporters/tensorrt_exporter.py +++ b/deployment/export/exporters/tensorrt_exporter.py @@ -61,7 +61,7 @@ def export( # Load any custom plugin .so libraries (e.g. the BEVFusion spconv ImplicitGemm plugin) # before the built-in plugin init, so their creators are registered for engine build. # No-op when plugin_libraries is empty (e.g. CenterPoint). - load_tensorrt_plugin_libraries(logger, self.config.plugin_libraries) + load_tensorrt_plugin_libraries(self.config.plugin_libraries) trt.init_libnvinfer_plugins(trt_logger, "") builder = trt.Builder(trt_logger) diff --git a/deployment/export/pipelines/component_builder.py b/deployment/export/pipelines/component_builder.py index 67640de79..1a712cd54 100644 --- a/deployment/export/pipelines/component_builder.py +++ b/deployment/export/pipelines/component_builder.py @@ -13,12 +13,15 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, List +from typing import TYPE_CHECKING, Any, Callable, List, Tuple import torch from deployment.config.schema import ComponentsConfig +if TYPE_CHECKING: + import onnx + @dataclass(frozen=True) class ExportableComponent: @@ -29,11 +32,17 @@ class ExportableComponent: config lookup, output filename, and logs. module: PyTorch module to export. sample_input: Sample input tensor for tracing. + post_transforms: Optional ONNX graph transforms applied (in order) to the exported + file after ``torch.onnx.export``. Each takes the loaded ``onnx.ModelProto`` and + returns the (possibly mutated) model. Empty for models needing no post-processing + (e.g. CenterPoint); used by BEVFusion for the TopK-constant fix and ImplicitGemm+ReLU + fusion. """ name: str module: torch.nn.Module sample_input: Any + post_transforms: Tuple[Callable[["onnx.ModelProto"], "onnx.ModelProto"], ...] = () class ModelComponentBuilder(ABC): diff --git a/deployment/export/pipelines/onnx_export_pipeline.py b/deployment/export/pipelines/onnx_export_pipeline.py index 2f4741e59..523a8e831 100644 --- a/deployment/export/pipelines/onnx_export_pipeline.py +++ b/deployment/export/pipelines/onnx_export_pipeline.py @@ -20,8 +20,9 @@ import logging from pathlib import Path -from typing import Any, List, Optional, Type +from typing import Any, Callable, List, Optional, Tuple, Type +import onnx import torch from deployment.config.base import BaseDeploymentConfig @@ -34,6 +35,11 @@ logger = logging.getLogger(__name__) +# A pipeline-level finalize hook: called once after every component has been exported, with the +# list of written ONNX paths, the output directory, and the deploy config. Used for cross-component +# post-processing such as BEVFusion's split (sparse+dense) -> single merged ONNX merge. +FinalizeHook = Callable[[List[str], Path, BaseDeploymentConfig], None] + class OnnxExportPipeline: """Model-agnostic ONNX export pipeline (one ONNX file per component). @@ -50,6 +56,7 @@ def __init__( sample_extractor: SampleExtractor, component_builder: ModelComponentBuilder, onnx_wrapper_cls: Type[BaseModelWrapper] = IdentityWrapper, + finalize: Optional[FinalizeHook] = None, ) -> None: """Initialize the pipeline. @@ -59,10 +66,14 @@ def __init__( exportable components. onnx_wrapper_cls: Model wrapper applied before ONNX export (defaults to ``IdentityWrapper`` for models needing no output reshaping). + finalize: Optional cross-component post-processing hook, called once after all + components are exported (see :data:`FinalizeHook`). ``None`` (the default) means + the exported per-component files are the final artifacts. """ self.sample_extractor = sample_extractor self.component_builder = component_builder self._onnx_wrapper_cls = onnx_wrapper_cls + self._finalize = finalize def export( self, @@ -99,6 +110,11 @@ def export( logger.info("=" * 80) exported_paths = self._export_components(components, output_dir_path, config) + + if self._finalize is not None: + logger.info("Running post-export finalize hook (%s)...", getattr(self._finalize, "__name__", "finalize")) + self._finalize(exported_paths, output_dir_path, config) + self._log_summary(exported_paths) return Artifact(path=str(output_dir_path)) @@ -158,10 +174,39 @@ def _export_components( logger.error("Failed to export %s", component.name, exc_info=True) raise RuntimeError(f"{component.name} ONNX export failed") from exc + if component.post_transforms: + self._apply_post_transforms(output_path, component.post_transforms, component.name) + exported_paths.append(str(output_path)) return exported_paths + @staticmethod + def _apply_post_transforms( + output_path: Path, + transforms: Tuple[Callable[[onnx.ModelProto], onnx.ModelProto], ...], + component_name: str, + ) -> None: + """Load the exported ONNX, apply each graph transform in order, and save it back. + + Args: + output_path: Path of the freshly exported ONNX file to transform in place. + transforms: Ordered graph transforms; each takes and returns an ``onnx.ModelProto``. + component_name: Component name, used only for logging. + + Raises: + RuntimeError: If loading, transforming, or saving the ONNX fails. + """ + try: + model_proto = onnx.load(str(output_path)) + for transform in transforms: + model_proto = transform(model_proto) + onnx.save_model(model_proto, str(output_path)) + except Exception as exc: + logger.error("Post-export transform failed for %s", component_name, exc_info=True) + raise RuntimeError(f"{component_name} ONNX post-export transform failed") from exc + logger.info("Applied %s post-export transform(s) to %s", len(transforms), output_path.name) + def _build_onnx_exporter(self, config: BaseDeploymentConfig, component_name: str) -> ONNXExporter: """Create an ONNX exporter for the given component. diff --git a/deployment/inference/base_inference_pipeline.py b/deployment/inference/base_inference_pipeline.py index 7d47f8fa5..dbf565d60 100644 --- a/deployment/inference/base_inference_pipeline.py +++ b/deployment/inference/base_inference_pipeline.py @@ -3,13 +3,14 @@ import logging import time from abc import ABC, abstractmethod -from typing import Any, Dict, Mapping, Optional, Tuple +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union +import numpy as np import torch from deployment.config.enums import Backend -from deployment.evaluation.evaluator_types import InferenceResult from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import InferenceResult logger = logging.getLogger(__name__) @@ -48,17 +49,60 @@ def torch_device(self) -> torch.device: """Return torch.device converted from canonical DeviceSpec.""" return self.device.to_torch_device() + def to_device_tensor(self, data: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + """Convert an array/tensor to a tensor on the pipeline's device.""" + if isinstance(data, np.ndarray): + data = torch.from_numpy(data) + return data.to(self.torch_device) + + def to_numpy(self, data: torch.Tensor, dtype: np.dtype = np.float32) -> np.ndarray: + """Convert a tensor to a contiguous numpy array of ``dtype``.""" + arr = data.cpu().numpy().astype(dtype) + if not arr.flags["C_CONTIGUOUS"]: + arr = np.ascontiguousarray(arr) + return arr + + @staticmethod + def order_outputs_by_config( + actual_names: Sequence[str], + expected_names: Sequence[str], + *, + strict: bool = True, + ) -> List[str]: + """Return output names in the config's declared order. + + ONNX/TensorRT may report outputs in arbitrary order, but postprocess depends on the + exact order declared in the component config. This returns the config order. + + Args: + actual_names: Output names reported by the runtime session/engine. + expected_names: Output names in the config's declared order. + strict: If True, raise when the two name sets differ (any missing/extra). If False, + return the expected names that are present (in order), then append any extras. + + Raises: + ValueError: If ``strict`` and the name sets do not match exactly. + """ + if strict: + expected_set, actual_set = set(expected_names), set(actual_names) + missing = expected_set - actual_set + extra = actual_set - expected_set + if missing or extra: + raise ValueError( + f"Output name mismatch: missing={sorted(missing)}, extra={sorted(extra)}; " + f"expected={sorted(expected_set)}, got={sorted(actual_set)}." + ) + return list(expected_names) + ordered = [n for n in expected_names if n in actual_names] + ordered += [n for n in actual_names if n not in ordered] + return ordered + @abstractmethod - def preprocess(self, input_data: Any) -> Tuple[Any, Dict[str, Any]]: + def preprocess(self, input_data: Any) -> Any: """Convert raw input into model-ready tensors/arrays. Returns: - A 2-tuple ``(model_input, preprocess_metadata)``: - - ``model_input``: Tensors or structure consumed by :meth:`run_model`. - - ``preprocess_metadata``: Dict merged into the ``metadata`` argument of - :meth:`infer` (together with any ``metadata`` passed by the caller) and - then passed to :meth:`postprocess`. Use an empty dict when nothing extra - is needed. + ``model_input``: Tensors or structure consumed by :meth:`run_model`. """ raise NotImplementedError @@ -71,7 +115,7 @@ def run_model(self, preprocessed_input: Any) -> Tuple[Any, Dict[str, float]]: - ``model_output``: Raw tensors or structure for :meth:`postprocess` (or returned as-is when ``infer(..., return_raw_outputs=True)``). - ``stage_latencies``: Per-substage timings in milliseconds; merged into - `~deployment.evaluation.evaluator_types.InferenceResult` + `~deployment.primitives.evaluator_types.InferenceResult` ``breakdown`` (e.g. ``voxel_encoder_ms``). """ raise NotImplementedError @@ -86,8 +130,8 @@ def postprocess( Args: model_output: Value returned by :meth:`run_model` (first element of its tuple). - metadata: Merged dict from ``infer(..., metadata=...)`` plus - ``preprocess_metadata`` from :meth:`preprocess`. May be empty. + metadata: Dict passed by the caller via ``infer(..., metadata=...)``. + May be empty. """ raise NotImplementedError @@ -99,11 +143,11 @@ def infer( Flow: 1) preprocess(input_data) 2) run_model(model_input) - 3) postprocess(model_output, merged_metadata) unless `return_raw_outputs=True` + 3) postprocess(model_output, metadata) unless `return_raw_outputs=True` Args: input_data: Raw input sample(s) in a project-defined format. - metadata: Optional auxiliary context merged with preprocess metadata. + metadata: Optional auxiliary context passed through to :meth:`postprocess`. return_raw_outputs: If True, skip `postprocess` and return raw model output. Returns: @@ -114,11 +158,11 @@ def infer( try: # Preprocess start_time = time.perf_counter() - model_input, preprocess_metadata = self.preprocess(input_data) + model_input = self.preprocess(input_data) preprocess_time = time.perf_counter() latency_breakdown["preprocessing_ms"] = (preprocess_time - start_time) * 1000 - # Build a new dict - metadata = {**(metadata or {}), **preprocess_metadata} + # Build a new dict from caller-provided metadata (passed to postprocess). + metadata = dict(metadata or {}) # Run model model_start = time.perf_counter() diff --git a/deployment/inference/gpu_resource_mixin.py b/deployment/inference/gpu_resource_mixin.py index 15fafff13..8f284df2a 100644 --- a/deployment/inference/gpu_resource_mixin.py +++ b/deployment/inference/gpu_resource_mixin.py @@ -25,12 +25,24 @@ class GPUResourceMixin(ABC): """ _cleanup_called: bool = False + # Free the CUDA cache every N samples during long eval loops (GPU backends only). + _gpu_cleanup_interval: int = 10 @abstractmethod def _release_gpu_resources(self) -> None: """Release backend-specific GPU resources owned by the instance.""" raise NotImplementedError + def periodic_cleanup(self, sample_idx: int) -> None: + """Free the CUDA cache every ``_gpu_cleanup_interval`` samples during long eval loops. + + Overrides the no-op :meth:`BaseInferencePipeline.periodic_cleanup` for every GPU-backed + pipeline that mixes in this class (TensorRT), so CUDA cache growth over a long evaluation + loop is bounded without each backend re-implementing the same guard. + """ + if sample_idx > 0 and sample_idx % self._gpu_cleanup_interval == 0 and torch.cuda.is_available(): + torch.cuda.empty_cache() + def cleanup(self) -> None: """Release GPU resources once and clear CUDA caches (best effort).""" if self._cleanup_called: diff --git a/deployment/inference/tensorrt_runner.py b/deployment/inference/tensorrt_runner.py new file mode 100644 index 000000000..31e2ee87d --- /dev/null +++ b/deployment/inference/tensorrt_runner.py @@ -0,0 +1,178 @@ +"""Shared TensorRT engine runner. + +One battle-tested implementation of the TensorRT run loop — allocate device buffers, +copy host->device, execute with CUDA-event timing, copy device->host — reused by every +per-project TensorRT pipeline (BEVFusion, CenterPoint). Project pipelines keep only their +model-specific pieces (which engine, how to name/order inputs and outputs); the GPU plumbing +lives here so it cannot drift between backends. +""" + +from __future__ import annotations + +import logging +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pycuda.driver as cuda +import tensorrt as trt + +from deployment.inference.gpu_resource_mixin import TensorRTResourceManager + +logger = logging.getLogger(__name__) + + +def list_trt_io_names(engine: trt.ICudaEngine) -> Tuple[List[str], List[str]]: + """Return ``(input_names, output_names)`` in TensorRT tensor-index order. + + Shared by every per-project TensorRT pipeline so input/output discovery cannot drift + between backends. + """ + inputs: List[str] = [] + outputs: List[str] = [] + for i in range(engine.num_io_tensors): + name = engine.get_tensor_name(i) + if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: + inputs.append(name) + else: + outputs.append(name) + return inputs, outputs + + +def load_trt_engine( + runtime: trt.Runtime, + engine_path: str, + *, + component_name: Optional[str] = None, +) -> Tuple[trt.ICudaEngine, trt.IExecutionContext]: + """Deserialize a TensorRT engine and create its execution context, failing loud. + + One implementation of the deserialize -> null-check -> create-context -> null-check + boilerplate, reused by every per-project TensorRT pipeline so the error messages (and the + OOM hint) stay identical across backends. + + Args: + runtime: TensorRT runtime used to deserialize the engine. + engine_path: Path to the serialized ``.engine`` file. + component_name: Optional component label for error messages (defaults to ``engine_path``). + + Returns: + Tuple of (engine, execution context). + + Raises: + RuntimeError: If deserialization or context creation fails (context failure is usually OOM). + """ + label = component_name or engine_path + with open(engine_path, "rb") as f: + engine = runtime.deserialize_cuda_engine(f.read()) + if engine is None: + raise RuntimeError(f"Failed to deserialize TensorRT engine: {engine_path}") + + context = engine.create_execution_context() + if context is None: + raise RuntimeError(f"Failed to create TensorRT execution context for {label} (likely GPU out-of-memory).") + + return engine, context + + +def _trt_dtype_to_numpy(trt_dtype: trt.DataType) -> np.dtype: + """Return the numpy dtype matching a TensorRT dtype, for correctly sized host buffers. + + Delegates to TensorRT's own ``nptype`` mapping and lets it raise for a dtype it cannot map: + failing loud is safer than guessing a size (e.g. defaulting to float32), which would + mis-size the GPU buffer and silently corrupt the data. + """ + return np.dtype(trt.nptype(trt_dtype)) + + +def _cast_to_binding_dtype(engine: trt.ICudaEngine, tensor_name: str, arr: np.ndarray) -> np.ndarray: + """Return ``arr`` as a C-contiguous buffer whose dtype matches the engine binding. + + Matching the binding dtype is critical for FP16 engines: split sparse ONNX is often traced + with FP32 voxels, but TensorRT ``fp16`` builds may bind ``voxels`` as ``HALF``. Feeding + float32 nbytes into a HALF binding misaligns the GPU buffer and corrupts the first + ImplicitGemm inputs (lidar_bev explosion while numpy voxel stats still look sane). Returns + ``arr`` unchanged when it already matches the binding dtype and is contiguous. + """ + binding_dtype = engine.get_tensor_dtype(tensor_name) + target_dtype = _trt_dtype_to_numpy(binding_dtype) + if arr.dtype != target_dtype: + logger.info( + "[trt-io] casting host buffer for tensor %r: numpy %s → %s (engine binding %s)", + tensor_name, + arr.dtype, + target_dtype, + binding_dtype, + ) + arr = np.asarray(arr, dtype=target_dtype) + if not arr.flags["C_CONTIGUOUS"]: + arr = np.ascontiguousarray(arr) + return arr + + +def run_trt_engine( + engine: trt.ICudaEngine, + context: trt.IExecutionContext, + inputs_by_name: Dict[str, np.ndarray], + output_names: List[str], +) -> Tuple[Dict[str, np.ndarray], float]: + """Run one engine end-to-end and return (outputs-by-name, pure-GPU time in ms). + + Handles all the dtype bookkeeping so callers pass plain arrays: input buffers are cast to + each binding's dtype and output buffers are allocated with the engine's actual output dtype, + so the same code path serves FP32 and FP16 engines. Timing uses CUDA events bracketing only + ``execute_async_v3`` on one stream, so the returned time is the engine's GPU compute and + excludes the H2D/D2H copies; it is read back while that stream is still alive. + + Args: + engine: TensorRT engine that owns ``context`` (needed to query binding dtypes/shapes). + context: Execution context; input shapes are set from ``inputs_by_name``. + inputs_by_name: Engine input tensor name -> host ndarray. A single-input engine is just a + one-entry map. + output_names: Engine output tensor names, in the desired return order. + + Returns: + Tuple of (outputs-by-name as host ndarrays, pure-GPU time in ms). + + Raises: + RuntimeError: If ``execute_async_v3`` reports a failure status. + """ + inputs_by_name = {name: _cast_to_binding_dtype(engine, name, arr) for name, arr in inputs_by_name.items()} + for name, arr in inputs_by_name.items(): + context.set_input_shape(name, arr.shape) + + # Output shapes can depend on the input shape, so read them only after set_input_shape. + host_outputs: Dict[str, np.ndarray] = {} + for name in output_names: + shape = context.get_tensor_shape(name) + arr = np.empty(shape, dtype=_trt_dtype_to_numpy(engine.get_tensor_dtype(name))) + if not arr.flags["C_CONTIGUOUS"]: + arr = np.ascontiguousarray(arr) + host_outputs[name] = arr + + with TensorRTResourceManager() as resources: + device_inputs = {name: resources.allocate(arr.nbytes) for name, arr in inputs_by_name.items()} + device_outputs = {name: resources.allocate(arr.nbytes) for name, arr in host_outputs.items()} + stream = resources.stream + + for name, arr in inputs_by_name.items(): + context.set_tensor_address(name, int(device_inputs[name])) + cuda.memcpy_htod_async(device_inputs[name], arr, stream) + + for name in output_names: + context.set_tensor_address(name, int(device_outputs[name])) + + start_event = cuda.Event() + end_event = cuda.Event() + start_event.record(stream) + succeeded = context.execute_async_v3(stream_handle=stream.handle) + if not succeeded: + raise RuntimeError("TensorRT execute_async_v3 returned failure status.") + end_event.record(stream) + + for name in output_names: + cuda.memcpy_dtoh_async(host_outputs[name], device_outputs[name], stream) + + resources.synchronize() + gpu_time_ms = float(end_event.time_since(start_event)) + + return host_outputs, gpu_time_ms diff --git a/deployment/io/mmdet3d_model.py b/deployment/io/mmdet3d_model.py new file mode 100644 index 000000000..2a7066e42 --- /dev/null +++ b/deployment/io/mmdet3d_model.py @@ -0,0 +1,50 @@ +"""Shared mmdet3d model-building core for deployment loaders. + +Holds the one invariant every project's export loader shares: build the model from an +MMEngine config, load the checkpoint, move it to the target device, put it in eval mode, +and stash the build config on ``model.cfg`` so callers can recover it. Project-specific +concerns (config transforms, module registration, post-load graph fusions) stay in each +project's own loader. +""" + +from __future__ import annotations + +import copy + +import torch +from mmengine.config import Config +from mmengine.registry import MODELS, init_default_scope +from mmengine.runner import load_checkpoint + +from deployment.primitives.device import DeviceSpec + + +def build_mmdet3d_model( + model_cfg: Config, + checkpoint_path: str, + device: DeviceSpec, +) -> torch.nn.Module: + """Build an mmdet3d model from config, load its checkpoint, and return it in eval mode. + + The project's module variants must already be registered with MMDet3D. Each project's + loader does this by importing its module packages at import time (``import ... # noqa: + F401``), so the registration is in place before this function is called. + + Args: + model_cfg: MMEngine model configuration whose ``model`` subtree is built. + checkpoint_path: Path to the ``.pth`` checkpoint file. + device: Target device specification. + + Returns: + The loaded model in eval mode, with ``model.cfg`` set to ``model_cfg`` so callers can + recover the config it was built from. + """ + init_default_scope("mmdet3d") + + model = MODELS.build(copy.deepcopy(model_cfg.model)) + torch_device = device.to_torch_device() + model.to(torch_device) + load_checkpoint(model, checkpoint_path, map_location=torch_device) + model.eval() + model.cfg = model_cfg + return model diff --git a/deployment/io/point_cloud_data_loader.py b/deployment/io/point_cloud_data_loader.py new file mode 100644 index 000000000..271039c87 --- /dev/null +++ b/deployment/io/point_cloud_data_loader.py @@ -0,0 +1,93 @@ +"""Shared MMDet3D point-cloud data loader for deployment. + +Point-cloud detectors (CenterPoint, BEVFusion, …) all wrap the same MMDet3D test dataset: +build it from ``model_cfg.test_dataloader.dataset``, run the pipeline once per sample, and +return ``points`` / ``metainfo`` / ``ground_truth``. Because that flow is identical, both +projects use this loader directly instead of subclassing it. + +TODO(vividf): BEVFusion is multi-modal (lidar + camera) but is deployed lidar-only today, so +this point-cloud loader is sufficient. Once camera inputs are added, give BEVFusion its own +loader (or extend this one) to load images and the lidar2img transforms alongside ``points``. +""" + +from __future__ import annotations + +import copy +from typing import Type + +import mmdet3d.datasets.transforms # noqa: F401 - registers transforms in the mmdet3d registry +import torch +from mmengine.config import Config +from mmengine.registry import DATASETS, init_default_scope +from typing_extensions import override + +from deployment.io.base_data_loader import BaseDataLoader, SampleData + + +class PointCloudDataLoader(BaseDataLoader): + """DataLoader that runs the MMDet3D test pipeline once per sample. + + The runtime payload is always ``points`` / ``metainfo`` (+ ``ground_truth`` for the loaded + sample); :attr:`sample_cls` / :attr:`model_input_cls` are the typed dicts describing it and + can be overridden if a project needs a richer typed payload. + """ + + #: Typed payloads returned by ``load_sample`` / ``preprocess`` (override in subclasses). + sample_cls: Type[SampleData] = SampleData + model_input_cls: Type[SampleData] = SampleData + + def __init__(self, model_cfg: Config, info_file: str = "") -> None: + """Build the MMDet3D dataset used for deployment evaluation. + + Args: + model_cfg: MMEngine model config; must have ``test_dataloader.dataset``. + info_file: Optional eval info file overriding the dataset's ``ann_file``; empty + keeps the model config's own ``ann_file``. + """ + super().__init__() + self.model_cfg = model_cfg + self.info_file = info_file + self.dataset = self._build_dataset(model_cfg, info_file) + + def _build_dataset(self, model_cfg: Config, info_file: str) -> torch.utils.data.Dataset: + init_default_scope("mmdet3d") + if not hasattr(model_cfg, "test_dataloader"): + raise ValueError("model_cfg must have 'test_dataloader' with dataset config") + dataset_cfg = copy.deepcopy(model_cfg.test_dataloader.dataset) + # Only override the eval info file when a deploy config supplies one; otherwise fall back + # to the model config's own ``ann_file``. + if info_file: + dataset_cfg["ann_file"] = info_file + dataset_cfg["test_mode"] = True + return DATASETS.build(dataset_cfg) + + @override + def load_sample(self, index: int) -> SampleData: + if index >= len(self.dataset): + raise IndexError(f"Sample index {index} out of range (0-{len(self.dataset)-1})") + + data = self.dataset[index] + points_tensor = data["inputs"]["points"].to("cpu") + if points_tensor.ndim != 2: + raise ValueError(f"Expected points tensor with shape [N, features], got {points_tensor.shape}") + + data_samples = data["data_samples"] + if data_samples is None: + raise ValueError("Dataset sample contains None 'data_samples', cannot build evaluation ground truth.") + + metainfo = getattr(data_samples, "metainfo", None) + eval_ann_info = getattr(data_samples, "eval_ann_info", None) + return self.sample_cls( + points=points_tensor, + metainfo=dict(metainfo) if metainfo else {}, + ground_truth=dict(eval_ann_info) if eval_ann_info else {}, + ) + + @override + def preprocess(self, sample: SampleData) -> SampleData: + return self.model_input_cls(points=sample["points"], metainfo=sample["metainfo"]) + + @property + @override + def num_samples(self) -> int: + return len(self.dataset) diff --git a/deployment/primitives/artifacts.py b/deployment/primitives/artifacts.py index 3903393aa..3adf9c275 100644 --- a/deployment/primitives/artifacts.py +++ b/deployment/primitives/artifacts.py @@ -12,14 +12,10 @@ from __future__ import annotations -import logging from dataclasses import dataclass from pathlib import Path from typing import Any, Mapping, Optional, Protocol, Union, runtime_checkable -logger = logging.getLogger(__name__) - - # ============================================================================ # Artifact Dataclass # ============================================================================ diff --git a/deployment/evaluation/evaluator_types.py b/deployment/primitives/evaluator_types.py similarity index 100% rename from deployment/evaluation/evaluator_types.py rename to deployment/primitives/evaluator_types.py diff --git a/deployment/primitives/tensorrt_plugins.py b/deployment/primitives/tensorrt_plugins.py index f49dab9eb..67f7c2ae7 100644 --- a/deployment/primitives/tensorrt_plugins.py +++ b/deployment/primitives/tensorrt_plugins.py @@ -10,6 +10,8 @@ import os from typing import Iterable, List, Tuple +logger = logging.getLogger(__name__) + _LOADED_PLUGIN_LIBS: set[str] = set() @@ -69,31 +71,31 @@ def _get_tensorrt_registries(): return uniq -def load_tensorrt_plugin_libraries( - logger: logging.Logger, - plugin_libraries: Iterable[str], -) -> Tuple[str, ...]: +def load_tensorrt_plugin_libraries(plugin_libraries: Iterable[str]) -> Tuple[str, ...]: """Load custom TensorRT plugin libraries. Paths are taken only from deploy_config.tensorrt_config.plugin_libraries. For TensorRT 10+ (Plugin V3), prefer registry-based loading (`IPluginRegistry.load_library`) so creators are registered properly. - Fallback to `ctypes.CDLL(..., RTLD_GLOBAL)` for compatibility. + Always also `ctypes.CDLL(..., RTLD_GLOBAL)` so symbols stay globally visible for + dependent shared objects. Libraries already loaded in this process are skipped. Args: - logger: Logger used for informative diagnostics. plugin_libraries: Plugin library paths from deploy config (e.g. tensorrt_config.plugin_libraries). Returns: - Tuple of loaded plugin library identifiers. + Tuple of the library paths newly loaded by this call (empty if all were already loaded). Raises: FileNotFoundError: If a configured path includes a slash but does not exist. OSError: If dlopen fails for a provided library. """ resolved = _normalize_libraries(plugin_libraries) - loaded_now: List[str] = [] + if not resolved: + logger.debug("No custom TensorRT plugin libraries configured. Set tensorrt_config.plugin_libraries.") + return () + newly_loaded: List[str] = [] for library in resolved: if library in _LOADED_PLUGIN_LIBS: continue @@ -101,7 +103,7 @@ def load_tensorrt_plugin_libraries( if "/" in library and not os.path.exists(library): raise FileNotFoundError(f"TensorRT plugin library not found: {library}") - # Try TensorRT registry loader first (required by many TRT10 V3 plugins). + # Try the TensorRT registry loader first (required by many TRT10 V3 plugins). loaded_with_registry = False for reg_name, registry in _get_tensorrt_registries(): if not hasattr(registry, "load_library"): @@ -113,17 +115,12 @@ def load_tensorrt_plugin_libraries( except Exception as exc: # pragma: no cover - best effort fallback path logger.debug("%s registry load failed for %s: %s", reg_name, library, exc) - # Always ensure symbols are globally visible for dependent shared objects. + # Always CDLL so symbols are globally visible; only log when the registry path did not. + ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) if not loaded_with_registry: - ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) logger.info("Loaded TensorRT plugin library via ctypes: %s", library) - else: - ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) _LOADED_PLUGIN_LIBS.add(library) - loaded_now.append(library) - - if not resolved: - logger.debug("No custom TensorRT plugin libraries configured. Set tensorrt_config.plugin_libraries.") + newly_loaded.append(library) - return tuple(_LOADED_PLUGIN_LIBS) + return tuple(newly_loaded) diff --git a/deployment/projects/bevfusion/cli.py b/deployment/projects/bevfusion/cli.py deleted file mode 100644 index f7e67f1a5..000000000 --- a/deployment/projects/bevfusion/cli.py +++ /dev/null @@ -1,16 +0,0 @@ -"""BEVFusion CLI extensions.""" - -from __future__ import annotations - -import argparse - - -def add_args(parser: argparse.ArgumentParser) -> None: - """Register BEVFusion-specific CLI flags onto a project subparser.""" - parser.add_argument( - "--module", - type=str, - default="main_body", - choices=["main_body", "image_backbone", "camera_bev_only"], - help="Module to export (default: main_body)", - ) diff --git a/deployment/projects/bevfusion/config/deploy_config.py b/deployment/projects/bevfusion/config/deploy_config.py deleted file mode 100644 index a39e447bf..000000000 --- a/deployment/projects/bevfusion/config/deploy_config.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -BEVFusion Deployment Configuration - -Example deploy config for BEVFusion LiDAR-only (main_body module). -Adapt checkpoint_path, info_file, and shape profiles to your model. -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "work_dirs/bevfusion/bevfusion_epoch_30.pth" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="onnx", - work_dir="work_dirs/bevfusion_deployment", - onnx_path=None, -) - -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Component Configuration -# -# BEVFusion exports a single ONNX model (main_body) that takes -# voxels/coors/num_points_per_voxel and outputs bbox_pred/score/label_pred. -# ============================================================================ -components = dict( - bevfusion_main_body=dict( - onnx_file="bevfusion_lidar.onnx", - engine_file="bevfusion_lidar.engine", - io=dict( - inputs=[ - dict(name="voxels", dtype="float32"), - dict(name="coors", dtype="int32"), - dict(name="num_points_per_voxel", dtype="int32"), - ], - outputs=[ - dict(name="bbox_pred", dtype="float32"), - dict(name="score", dtype="float32"), - dict(name="label_pred", dtype="int64"), - ], - dynamic_axes={ - "voxels": {0: "voxels_num"}, - "coors": {0: "voxels_num"}, - "num_points_per_voxel": {0: "voxels_num"}, - }, - ), - tensorrt_profile=dict( - voxels=dict( - min_shape=[1, 10, 5], - opt_shape=[64000, 10, 5], - max_shape=[256000, 10, 5], - ), - coors=dict( - min_shape=[1, 3], - opt_shape=[64000, 3], - max_shape=[256000, 3], - ), - num_points_per_voxel=dict( - min_shape=[1], - opt_shape=[64000], - max_shape=[256000], - ), - ), - ), -) - -# ============================================================================ -# Runtime I/O settings -# ============================================================================ -runtime_io = dict( - info_file="info/t4dataset_j6gen2_base_infos_test.pkl", - sample_idx=0, -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=17, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -# BEVFusion ONNX uses ImplicitGemm / GetIndicePairsImplicitGemm (spconv). You must -# provide the plugin .so and list it here, or TensorRT export will fail with -# "Plugin not found". See: projects/BEVFusion/plugins/README.md -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=1 << 32, - # Optional: enable FP16 at build time via policy_flags=dict(FP16=True). - # Set this after placing libautoware_tensorrt_plugins.so in the image (e.g. under - # /opt/plugins/). Alternatively set env DEPLOY_TENSORRT_PLUGIN_LIBS. - plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"], - # plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"] -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=5, - num_warmup=2, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict[str, bool | str]( - enabled=False, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cuda", test_backend="onnx", test_device="cuda"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cuda", test_backend="onnx", test_device="cuda"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/bevfusion/entrypoint.py b/deployment/projects/bevfusion/entrypoint.py deleted file mode 100644 index ab6e130fa..000000000 --- a/deployment/projects/bevfusion/entrypoint.py +++ /dev/null @@ -1,174 +0,0 @@ -"""BEVFusion deployment entrypoint invoked by the unified CLI.""" - -from __future__ import annotations - -import argparse -import logging - -from mmengine.config import Config - -from deployment.cli.args import add_deployment_file_logging, setup_logging -from deployment.config.base import BaseDeploymentConfig -from deployment.export.contexts import ExportContext -from deployment.projects.bevfusion.evaluation.evaluator import BEVFusionEvaluator -from deployment.projects.bevfusion.evaluation.executor import BEVFusionExecutor -from deployment.projects.bevfusion.io.component_utils import ( - has_component, - is_split_bevfusion_components, - maybe_add_merged_main_body_component, - should_merge_split_bevfusion, -) -from deployment.projects.bevfusion.io.data_loader import BEVFusionDataLoader -from deployment.projects.bevfusion.runner import BEVFusionDeploymentRunner -from deployment.projects.registry import project_registry - - -def _validate_bevfusion_components(config: BaseDeploymentConfig) -> None: - if is_split_bevfusion_components(config.components_cfg): - config.components_cfg.get_component("bevfusion_sparse") - config.components_cfg.get_component("bevfusion_dense") - if has_component(config.components_cfg, "bevfusion_main_body"): - config.components_cfg.get_component("bevfusion_main_body") - else: - config.components_cfg.get_component("bevfusion_main_body") - - -def _apply_bevfusion_component_merge_overlay( - config: BaseDeploymentConfig, - deploy_cfg: Config, - logger: logging.Logger, -) -> None: - """Apply optional split+merge overlay driven by deploy config.""" - if not should_merge_split_bevfusion(deploy_cfg): - return - before_names = list(config.components_cfg.component_names()) - config.components_cfg = maybe_add_merged_main_body_component( - deploy_cfg=deploy_cfg, - components_cfg=config.components_cfg, - ) - after_names = list(config.components_cfg.component_names()) - logger.info( - "BEVFusion merge flag enabled: keeping split export and adding merged artifacts component (%s -> %s)", - before_names, - after_names, - ) - - -def _extract_metrics_config(model_cfg: Config, logger: logging.Logger): - """Extract Detection3DMetricsConfig from model config. - - Tries T4MetricV2 first; falls back to a basic config if a different (or no) evaluator - is configured, so non-T4MetricV2 model configs still evaluate. - """ - from deployment.metrics.detection_3d_metrics import Detection3DMetricsConfig, extract_t4metric_v2_config - - class_names = model_cfg.class_names - - def _cfg_get(obj, key, default=None): - if obj is None: - return default - if isinstance(obj, dict): - return obj.get(key, default) - if key in obj: - return obj[key] - return getattr(obj, key, default) - - evaluator_cfg = getattr(model_cfg, "val_evaluator", None) or getattr(model_cfg, "test_evaluator", None) - if evaluator_cfg is None: - logger.warning("No evaluator config found; using basic metrics config") - return Detection3DMetricsConfig(class_names=class_names, frame_id="base_link") - - evaluator_type = getattr(evaluator_cfg, "type", None) - - if evaluator_type == "T4MetricV2": - return extract_t4metric_v2_config(model_cfg) - - perception_cfg = _cfg_get(evaluator_cfg, "perception_evaluator_configs") - frame_id = _cfg_get(evaluator_cfg, "frame_id") or _cfg_get(perception_cfg, "frame_id") or "base_link" - - logger.info( - "Evaluator type '%s'; using Detection3DMetricsConfig fallback (frame_id=%s)", - evaluator_type, - frame_id, - ) - return Detection3DMetricsConfig(class_names=class_names, frame_id=frame_id) - - -def _apply_spconv_do_sort(deploy_cfg: Config, logger: logging.Logger) -> None: - """Apply the ``spconv_do_sort`` field from ``deploy_cfg`` (default ``True``) to the - GetIndicePairsImplicitGemm symbolic/forward path. - - Controls the pair-mask argsort baked into the exported sparse graph; set ``False`` in a - deploy config to skip it. - """ - value = bool(deploy_cfg.get("spconv_do_sort", True)) - from projects.SparseConvolution.sparse_functional import set_do_sort - - set_do_sort(value) - logger.info( - "spconv_do_sort: %s (baked into GetIndicePairsImplicitGemm.do_sort_i at ONNX export)", - value, - ) - - -def run(args: argparse.Namespace) -> int: - """Run the BEVFusion deployment workflow.""" - deploy_cfg = Config.fromfile(args.deploy_cfg) - logger = setup_logging(args.log_level) - model_cfg = Config.fromfile(args.model_cfg) - config = BaseDeploymentConfig(deploy_cfg) - _apply_bevfusion_component_merge_overlay(config, deploy_cfg, logger) - - log_file = config.resolved_deploy_log_file - if log_file: - add_deployment_file_logging(log_file) - logger.info("Deployment log file: %s", log_file) - - project_registry.validate_required_components("bevfusion", config.components_cfg) - _validate_bevfusion_components(config) - _apply_spconv_do_sort(deploy_cfg, logger) - - logger.info("=" * 80) - logger.info("BEVFusion Deployment Pipeline") - logger.info("=" * 80) - - info_file = (deploy_cfg.get("runtime_io", {}) or {}).get("info_file", "") - data_loader = BEVFusionDataLoader( - info_file=info_file, - model_cfg=model_cfg, - ) - logger.info("Loaded %s samples", data_loader.num_samples) - - metrics_config = _extract_metrics_config(model_cfg, logger) - - plugin_libraries = tuple((deploy_cfg.get("tensorrt_config", {}) or {}).get("plugin_libraries", ()) or ()) - - # One executor instance, shared by the evaluator (evaluate/verify) and the runner - # (which hands it the loaded reference model after export). - executor = BEVFusionExecutor( - components_cfg=config.components_cfg, - tensorrt_plugin_libraries=plugin_libraries, - ) - - evaluator = BEVFusionEvaluator( - model_cfg=model_cfg, - metrics_config=metrics_config, - executor=executor, - ) - - module = getattr(args, "module", "main_body") - - runner = BEVFusionDeploymentRunner( - data_loader=data_loader, - evaluator=evaluator, - executor=executor, - config=config, - model_cfg=model_cfg, - deploy_cfg=deploy_cfg, - module=module, - plugin_libraries=plugin_libraries, - ) - - context = ExportContext() - runner.run(context=context) - return 0 diff --git a/deployment/projects/bevfusion/evaluation/evaluator.py b/deployment/projects/bevfusion/evaluation/evaluator.py deleted file mode 100644 index 94eaeb8f0..000000000 --- a/deployment/projects/bevfusion/evaluation/evaluator.py +++ /dev/null @@ -1,107 +0,0 @@ -"""BEVFusion evaluator for deployment. - -Thin subclass of ``Detection3DEvaluator``: only ``print_results`` (BEVFusion's indented -sparse/dense stage-wise latency layout) is BEVFusion-specific; the metrics hooks -(parse/accumulate/build/summarize) are shared with the base -(see ``deployment.evaluation.detection3d_evaluator``). -""" - -from __future__ import annotations - -import logging -from typing import Dict, Tuple - -from typing_extensions import override - -from deployment.evaluation.base_evaluator import EvalResultDict -from deployment.evaluation.detection3d_evaluator import Detection3DEvaluator - -logger = logging.getLogger(__name__) - -# (stage_key, indent_level): indent 0 = top-level; each +1 adds one leading space before the label. -_BEVFUSION_LATENCY_STAGE_LAYOUT: Tuple[Tuple[str, int], ...] = ( - ("preprocessing_ms", 0), - ("model_ms", 0), - ("bevfusion_ms", 0), - ("sparse_encoder_ms", 1), - ("dense_engine_ms", 1), - ("voxel_encoder_ms", 1), - ("backbone_ms", 2), - ("neck_ms", 2), - ("head_ms", 2), - ("post_scoring_ms", 2), - ("dense_unattributed_ms", 2), - ("postprocessing_ms", 0), -) - -_BEVFUSION_STAGE_DISPLAY_NAME: Dict[str, str] = { - "preprocessing_ms": "Preprocessing", - "model_ms": "Model", - "postprocessing_ms": "Postprocessing", - "bevfusion_ms": "Bevfusion", - "sparse_encoder_ms": "Sparse Encoder", - "dense_engine_ms": "Dense Engine", - "voxel_encoder_ms": "Voxel Encoder", - "backbone_ms": "Backbone", - "neck_ms": "Neck", - "head_ms": "Head", - "post_scoring_ms": "Post Scoring", - "dense_unattributed_ms": "Dense Unattributed", -} - - -def _bevfusion_stage_display_name(stage_key: str) -> str: - return _BEVFUSION_STAGE_DISPLAY_NAME.get( - stage_key, - stage_key.replace("_ms", "").replace("_", " ").title(), - ) - - -class BEVFusionEvaluator(Detection3DEvaluator): - """Evaluator for BEVFusion 3D detection deployment.""" - - @override - def print_results(self, results: EvalResultDict) -> None: - """Log the metrics report, latency statistics, and BEVFusion's indented breakdown.""" - metrics_report = self.metrics_interface.format_metrics_report() - if metrics_report: - for line in metrics_report.rstrip().split("\n"): - logger.info(line) - - if "latency" in results: - self._log_latency_stats(results) - - if "latency_breakdown" in results: - breakdown = results["latency_breakdown"] - breakdown_dict = breakdown.to_dict() if hasattr(breakdown, "to_dict") else breakdown - if breakdown_dict: - logger.info("") - logger.info("Stage-wise Latency Breakdown:") - printed: set[str] = set() - for stage_key, indent_level in _BEVFUSION_LATENCY_STAGE_LAYOUT: - if stage_key not in breakdown_dict: - continue - stats = breakdown_dict[stage_key] - stats_dict = stats.to_dict() if hasattr(stats, "to_dict") else stats - mean_ms = stats_dict.get("mean_ms", 0.0) - std_ms = stats_dict.get("std_ms", 0.0) - if mean_ms == 0.0 and std_ms == 0.0: - continue - printed.add(stage_key) - prefix = " " * (2 + indent_level) - label = _bevfusion_stage_display_name(stage_key) - logger.info("%s%-18s: %.2f ± %.2f ms", prefix, label, mean_ms, std_ms) - - extra_keys = sorted(k for k in breakdown_dict if k not in printed) - for stage_key in extra_keys: - stats = breakdown_dict[stage_key] - stats_dict = stats.to_dict() if hasattr(stats, "to_dict") else stats - mean_ms = stats_dict.get("mean_ms", 0.0) - std_ms = stats_dict.get("std_ms", 0.0) - if mean_ms == 0.0 and std_ms == 0.0: - continue - label = _bevfusion_stage_display_name(stage_key) - logger.info(" %-18s: %.2f ± %.2f ms", label, mean_ms, std_ms) - - logger.info("") - logger.info("Total Samples: %s", results["num_samples"]) diff --git a/deployment/projects/bevfusion/evaluation/executor.py b/deployment/projects/bevfusion/evaluation/executor.py deleted file mode 100644 index c2c1974e7..000000000 --- a/deployment/projects/bevfusion/evaluation/executor.py +++ /dev/null @@ -1,58 +0,0 @@ -"""BEVFusion backend executor. - -Thin subclass of ``PointDetectionExecutor``: declares the BEVFusion pipeline classes, the -split/merged output-name lookup, and forwards custom spconv ImplicitGemm ``plugin_libraries`` to the -TensorRT pipeline. Pipeline creation and ``(points, metainfo)`` input prep are shared with the -base (see ``deployment.evaluation.point_detection_executor``). - -This replaces the OLD ``BEVFusionPipelineFactory`` (the global pipeline registry was removed in -the refactor): pipeline construction uses the reference model on ``self.pytorch_model`` (set by -the runner after export). -""" - -from typing import Any, Iterable, List, Mapping, Optional - -from typing_extensions import override - -from deployment.config.schema import ComponentsConfig -from deployment.evaluation.point_detection_executor import PointDetectionExecutor -from deployment.projects.bevfusion.inference.onnx_inference_pipeline import BEVFusionONNXPipeline -from deployment.projects.bevfusion.inference.pytorch_inference_pipeline import BEVFusionPyTorchPipeline -from deployment.projects.bevfusion.inference.tensorrt_inference_pipeline import BEVFusionTensorRTPipeline -from deployment.projects.bevfusion.io.component_utils import has_component, is_split_bevfusion_components - - -class BEVFusionExecutor(PointDetectionExecutor): - """Backend execution primitives for BEVFusion (pipeline creation, input prep). - - Args: - components_cfg: Unified components configuration, forwarded to the ONNX/TensorRT - pipelines so they can resolve split (sparse+dense) vs merged main-body artifacts. - tensorrt_plugin_libraries: Custom TensorRT plugin ``.so`` paths forwarded to the - TensorRT pipeline (e.g. the spconv ImplicitGemm plugin); empty when none is needed. - """ - - task_name = "BEVFusion" - pytorch_pipeline_cls = BEVFusionPyTorchPipeline - onnx_pipeline_cls = BEVFusionONNXPipeline - tensorrt_pipeline_cls = BEVFusionTensorRTPipeline - - def __init__(self, components_cfg: ComponentsConfig, tensorrt_plugin_libraries: Iterable[str] = ()) -> None: - super().__init__(components_cfg) - self._tensorrt_plugin_libraries = tuple(tensorrt_plugin_libraries) - - @override - def _tensorrt_pipeline_kwargs(self) -> Mapping[str, Any]: - """Forward the custom spconv ImplicitGemm plugin ``.so`` paths to the TensorRT pipeline.""" - return {"plugin_libraries": self._tensorrt_plugin_libraries} - - @override - def get_output_names(self) -> Optional[List[str]]: - """Return the model output names (split→dense outputs; otherwise main-body outputs).""" - if is_split_bevfusion_components(self._components_cfg) and not has_component( - self._components_cfg, "bevfusion_main_body" - ): - comp = self._components_cfg.get_component("bevfusion_dense") - else: - comp = self._components_cfg.get_component("bevfusion_main_body") - return [out.name for out in comp.io.outputs] diff --git a/deployment/projects/bevfusion/export/onnx_export_pipeline.py b/deployment/projects/bevfusion/export/onnx_export_pipeline.py deleted file mode 100644 index 08ba2495b..000000000 --- a/deployment/projects/bevfusion/export/onnx_export_pipeline.py +++ /dev/null @@ -1,732 +0,0 @@ -"""BEVFusion ONNX export pipeline. - -Exports the BEVFusion main_body to a single ONNX file, including the TopK fix. -Replicates the logic from projects/BEVFusion/deploy/ within the new deployment framework. -""" - -from __future__ import annotations - -import contextlib -import logging -import os -import warnings -from pathlib import Path -from typing import Any, Dict, Optional - -import numpy as np -import onnx -import onnx_graphsurgeon as gs -import torch -import torch.nn as nn -import torch.nn.functional as F - -from deployment.config.base import BaseDeploymentConfig -from deployment.io.base_data_loader import BaseDataLoader -from deployment.primitives.artifacts import Artifact -from deployment.projects.bevfusion.io.component_utils import ( - has_component, - is_split_bevfusion_components, - should_merge_split_bevfusion, -) - -logger = logging.getLogger(__name__) - - -def _normalize_sparse_coors_for_autoware(coors: torch.Tensor) -> torch.Tensor: - """Normalize sparse coordinates to the legacy Autoware export contract. - - Graph **inputs** must be ``[z, y, x]`` (no batch). This wrapper flips to - ``[x, y, z]`` and prepends batch — same as ``projects/BEVFusion/deploy/containers.py``. - Voxelization outputs ``[x, y, z]``; convert with ``voxel_indices_xyz_to_graph_input_zyx`` - before tracing or feeding ONNX/TRT. - """ - from deployment.projects.bevfusion.io.coors_contract import graph_input_zyx_to_model_indices_xyz - - coors = coors.to(dtype=torch.int32) - if coors.shape[1] == 3: - num_points = coors.shape[0] - coors = graph_input_zyx_to_model_indices_xyz(coors) - batch_coors = torch.zeros(num_points, 1, dtype=torch.int32, device=coors.device) - coors = torch.cat([batch_coors, coors], dim=1).contiguous() - return coors - - -def _head_dict_to_export_outputs(outputs: dict) -> tuple: - """Turn the detection-head output dict into the (bbox_pred, score, label) ONNX outputs.""" - score = outputs["heatmap"].sigmoid() - one_hot = F.one_hot(outputs["query_labels"], num_classes=score.size(1)).permute(0, 2, 1) - score = score * outputs["query_heatmap_score"] * one_hot - score = score[0].max(dim=0)[0] - - bbox_pred = torch.cat( - [outputs["center"][0], outputs["height"][0], outputs["dim"][0], outputs["rot"][0], outputs["vel"][0]], - dim=0, - ) - - return bbox_pred, score, outputs["query_labels"][0] - - -class BEVFusionSparseWrapper(nn.Module): - """LiDAR sparse tower only: voxels/coors/num_points → BEV feature map.""" - - def __init__(self, model: nn.Module) -> None: - super().__init__() - self.mod = model - - def forward( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - ) -> torch.Tensor: - voxels = voxels.to(dtype=torch.float32) - coors = _normalize_sparse_coors_for_autoware(coors) - - return self.mod.extract_pts_feat(voxels, coors, num_points_per_voxel, points=None) - - -class BEVFusionDenseWrapper(nn.Module): - """SECOND + neck + head (+ ONNX postprocess). Input: ``lidar_bev`` [B,C,H,W].""" - - def __init__(self, model: nn.Module) -> None: - super().__init__() - self.mod = model - - def forward(self, lidar_bev: torch.Tensor) -> tuple: - x = lidar_bev - if self.mod.pts_backbone is not None: - x = self.mod.pts_backbone(x) - if self.mod.pts_neck is not None: - x = self.mod.pts_neck(x) - x = self.mod._align_lidar_bev_to_head_grid(x) - outputs = self.mod.bbox_head(x, []) - head_out = outputs[0][0] - return _head_dict_to_export_outputs(head_out) - - -class BEVFusionMainBodyWrapper(nn.Module): - """Wrapper for BEVFusion that matches the ONNX export interface. - - Takes voxels/coors/num_points_per_voxel and returns bbox_pred/score/label_pred. - Replicates TrtBevFusionMainContainer from projects/BEVFusion/deploy/containers.py. - """ - - def __init__(self, model: nn.Module) -> None: - super().__init__() - self.mod = model - - def forward( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - ) -> tuple: - # spconv requires int32 indices; float batch column (torch.zeros default) + int coors - # yields float tensor and can CUDA fault in implicit_gemm. Keep voxels FP32. - voxels = voxels.to(dtype=torch.float32) - coors = _normalize_sparse_coors_for_autoware(coors) - - batch_inputs_dict = { - "voxels": {"voxels": voxels, "coors": coors, "num_points_per_voxel": num_points_per_voxel}, - } - - outputs = self.mod._forward(batch_inputs_dict, using_image_features=True) - return _head_dict_to_export_outputs(outputs) - - -class BEVFusionONNXExportPipeline: - """ONNX export for BEVFusion. - - - **Single-file** (``bevfusion_main_body``): full graph, TopK fix applied. - - **Split** (``bevfusion_sparse`` + ``bevfusion_dense``): sparse tower ONNX + dense ONNX - (route 1: sparse can go to libspconv / plugin; dense → TensorRT without spconv ops). - """ - - def __init__( - self, - module: str = "main_body", - logger: Optional[logging.Logger] = None, - ) -> None: - self.module = module - self.logger = logger or logging.getLogger(__name__) - - def export( - self, - *, - model: torch.nn.Module, - data_loader: BaseDataLoader, - output_dir: str, - config: BaseDeploymentConfig, - sample_idx: int = 0, - ) -> Artifact: - output_dir_path = Path(output_dir) - output_dir_path.mkdir(parents=True, exist_ok=True) - - if is_split_bevfusion_components(config.components_cfg): - return self._export_split(model, data_loader, output_dir_path, config, sample_idx) - - self.logger.info("=" * 80) - self.logger.info("Exporting BEVFusion to ONNX (single-file)") - self.logger.info("=" * 80) - - device = next(model.parameters()).device - self.logger.info(f"Model device: {device}") - - component_cfg = config.components_cfg.get_component("bevfusion_main_body") - onnx_filename = component_cfg.onnx_file - output_path = output_dir_path / onnx_filename - temp_path = output_dir_path / onnx_filename.replace(".onnx", "_temp_to_be_fixed.onnx") - - self.logger.info(f"Loading sample {sample_idx} for export tracing...") - sample = data_loader.load_sample(sample_idx) - points = sample["points"] - self.logger.info(f"Sample loaded: points shape={points.shape}") - - self.logger.info("Running voxelization...") - voxels, coors, num_points_per_voxel = self._voxelize(model, points) - self.logger.info(f"Voxelization done: {voxels.shape[0]} voxels") - - onnx_cfg = self._get_onnx_config(config, "bevfusion_main_body") - self.logger.info( - f"ONNX config: opset={onnx_cfg['opset_version']}, inputs={onnx_cfg['input_names']}, outputs={onnx_cfg['output_names']}" - ) - - self.logger.info("Running torch.onnx.export...") - self._export_to_onnx( - model, - voxels, - coors, - num_points_per_voxel, - str(temp_path), - onnx_cfg, - fuse_spconv_bn=bool(config.deploy_cfg.get("fuse_spconv_bn", False)), - ) - - num_proposals = self._get_num_proposals(model) - self._fix_topk(str(temp_path), str(output_path), num_proposals) - - self.logger.info("=" * 80) - self.logger.info(f"BEVFusion ONNX export successful: {output_path}") - self.logger.info("=" * 80) - - return Artifact(path=str(output_dir_path)) - - def _export_split( - self, - model: torch.nn.Module, - data_loader: BaseDataLoader, - output_dir_path: Path, - config: BaseDeploymentConfig, - sample_idx: int, - ) -> Artifact: - """Export ``bevfusion_sparse.onnx`` and ``bevfusion_dense.onnx``.""" - self.logger.info("=" * 80) - self.logger.info("Exporting BEVFusion to ONNX (split: sparse + dense)") - self.logger.info("=" * 80) - - self._assert_split_model_ok(model) - - device = next(model.parameters()).device - self.logger.info(f"Model device: {device}") - - self.logger.info(f"Loading sample {sample_idx} for export tracing...") - sample = data_loader.load_sample(sample_idx) - points = sample["points"] - self.logger.info(f"Sample loaded: points shape={points.shape}") - - self.logger.info("Running voxelization...") - voxels, coors, num_points_per_voxel = self._voxelize(model, points) - self.logger.info(f"Voxelization done: {voxels.shape[0]} voxels") - - sparse_cfg = config.components_cfg.get_component("bevfusion_sparse") - dense_cfg = config.components_cfg.get_component("bevfusion_dense") - sparse_onnx = output_dir_path / sparse_cfg.onnx_file - dense_onnx = output_dir_path / dense_cfg.onnx_file - dense_temp = output_dir_path / dense_cfg.onnx_file.replace(".onnx", "_temp_to_be_fixed.onnx") - - onnx_cfg_sparse = self._get_onnx_config(config, "bevfusion_sparse") - onnx_cfg_dense = self._get_onnx_config(config, "bevfusion_dense") - - self.logger.info( - "Sparse ONNX: inputs=%s outputs=%s", - onnx_cfg_sparse["input_names"], - onnx_cfg_sparse["output_names"], - ) - self.logger.info("Running torch.onnx.export (sparse)...") - self._export_to_onnx( - model, - voxels, - coors, - num_points_per_voxel, - str(sparse_onnx), - onnx_cfg_sparse, - wrapper="sparse", - fuse_spconv_bn=bool(config.deploy_cfg.get("fuse_spconv_bn", False)), - ) - self._postprocess_sparse_onnx_fp(config=config, sparse_onnx_path=sparse_onnx) - - with torch.no_grad(): - sw = BEVFusionSparseWrapper(model) - sw.eval() - trace_dev = device - lidar_bev = sw( - voxels.to(trace_dev), coors.to(trace_dev, dtype=torch.int32), num_points_per_voxel.to(trace_dev) - ) - self.logger.info("Dense trace input lidar_bev shape: %s", tuple(lidar_bev.shape)) - - self.logger.info( - "Dense ONNX: inputs=%s outputs=%s", - onnx_cfg_dense["input_names"], - onnx_cfg_dense["output_names"], - ) - self.logger.info("Running torch.onnx.export (dense)...") - self._export_dense_to_onnx(model, lidar_bev, str(dense_temp), onnx_cfg_dense) - - num_proposals = self._get_num_proposals(model) - self._fix_topk(str(dense_temp), str(dense_onnx), num_proposals) - - if should_merge_split_bevfusion(config.deploy_cfg): - self._merge_split_onnx_artifact( - config=config, - sparse_onnx_path=sparse_onnx, - dense_onnx_path=dense_onnx, - output_dir_path=output_dir_path, - ) - - self.logger.info("=" * 80) - self.logger.info("Split ONNX export OK: %s , %s", sparse_onnx, dense_onnx) - self.logger.info("=" * 80) - - return Artifact(path=str(output_dir_path)) - - @staticmethod - def _deploy_cfg_fuse_implicit_gemm_relu(deploy_cfg: Any, *, default: bool = True) -> bool: - """Read ``spconv_fuse_implicit_gemm_relu`` (fuse trailing Relu into ImplicitGemm nodes).""" - val = deploy_cfg.get("spconv_fuse_implicit_gemm_relu", None) - if val is not None: - return bool(val) - return default - - def _postprocess_sparse_onnx_fp(self, *, config: BaseDeploymentConfig, sparse_onnx_path: Path) -> None: - """Optional FP sparse ONNX postprocess (ImplicitGemm activation fusion).""" - enable_fuse = self._deploy_cfg_fuse_implicit_gemm_relu(config.deploy_cfg, default=False) - if not enable_fuse: - self.logger.info("Sparse ONNX postprocess: ImplicitGemm ReLU fuse disabled by deploy config.") - return - if not sparse_onnx_path.exists(): - raise FileNotFoundError(f"Sparse ONNX not found for postprocess: {sparse_onnx_path}") - - from deployment.projects.bevfusion.export.onnx_fuse_implicit_gemm_activation import ( - fuse_autoware_implicit_gemm_trailing_relu, - ) - - model = onnx.load(str(sparse_onnx_path)) - n_relu = fuse_autoware_implicit_gemm_trailing_relu(model) - onnx.save_model(model, str(sparse_onnx_path)) - - self.logger.info( - "Sparse ONNX postprocess: ImplicitGemm fuse done (trailing Relu=%d): %s", - n_relu, - sparse_onnx_path, - ) - - def _merge_split_onnx_artifact( - self, - *, - config: BaseDeploymentConfig, - sparse_onnx_path: Path, - dense_onnx_path: Path, - output_dir_path: Path, - ) -> None: - """Merge split sparse+dense ONNX into single main_body ONNX.""" - if not has_component(config.components_cfg, "bevfusion_main_body"): - raise KeyError( - "bevfusion_merge is enabled but components_cfg has no 'bevfusion_main_body'. " - "Ensure merge overlay is applied before export." - ) - merged_cfg = config.components_cfg.get_component("bevfusion_main_body") - merged_path = output_dir_path / merged_cfg.onnx_file - - try: - from onnx import compose as onnx_compose - except Exception as e: - raise RuntimeError("ONNX compose utilities unavailable; cannot merge split ONNX.") from e - - if not sparse_onnx_path.exists(): - raise FileNotFoundError(f"Sparse ONNX not found: {sparse_onnx_path}") - if not dense_onnx_path.exists(): - raise FileNotFoundError(f"Dense ONNX not found: {dense_onnx_path}") - - sparse_model = onnx.load(str(sparse_onnx_path)) - dense_model = onnx.load(str(dense_onnx_path)) - - # onnx.compose.merge_models requires identical IR/opset metadata. - target_ir = max(int(sparse_model.ir_version), int(dense_model.ir_version)) - sparse_model.ir_version = target_ir - dense_model.ir_version = target_ir - - sparse_opsets = {imp.domain: int(imp.version) for imp in sparse_model.opset_import} - dense_opsets = {imp.domain: int(imp.version) for imp in dense_model.opset_import} - merged_opsets = dict(sparse_opsets) - for domain, version in dense_opsets.items(): - merged_opsets[domain] = max(version, merged_opsets.get(domain, version)) - merged_opset_ids = [onnx.helper.make_operatorsetid(d, v) for d, v in merged_opsets.items()] - del sparse_model.opset_import[:] - sparse_model.opset_import.extend(merged_opset_ids) - del dense_model.opset_import[:] - dense_model.opset_import.extend(merged_opset_ids) - - sparse_pref = onnx_compose.add_prefix(sparse_model, prefix="sparse/") - dense_pref = onnx_compose.add_prefix(dense_model, prefix="dense/") - - sparse_out_name = config.components_cfg.get_component("bevfusion_sparse").io.outputs[0].name - dense_in_name = config.components_cfg.get_component("bevfusion_dense").io.inputs[0].name - io_map = [(f"sparse/{sparse_out_name}", f"dense/{dense_in_name}")] - - merged_model = onnx_compose.merge_models(sparse_pref, dense_pref, io_map=io_map) - merged_graph = gs.import_onnx(merged_model) - - sparse_inputs = [inp.name for inp in config.components_cfg.get_component("bevfusion_sparse").io.inputs] - dense_outputs = [out.name for out in config.components_cfg.get_component("bevfusion_dense").io.outputs] - if len(merged_graph.inputs) != len(sparse_inputs): - self.logger.warning( - "Merged ONNX input count mismatch: graph=%d expected=%d", - len(merged_graph.inputs), - len(sparse_inputs), - ) - if len(merged_graph.outputs) != len(dense_outputs): - self.logger.warning( - "Merged ONNX output count mismatch: graph=%d expected=%d", - len(merged_graph.outputs), - len(dense_outputs), - ) - for i, name in enumerate(sparse_inputs): - if i < len(merged_graph.inputs): - merged_graph.inputs[i].name = name - for i, name in enumerate(dense_outputs): - if i < len(merged_graph.outputs): - merged_graph.outputs[i].name = name - - merged_graph.cleanup().toposort() - onnx.save_model(gs.export_onnx(merged_graph), str(merged_path)) - self.logger.info("Merged split ONNX -> %s", merged_path) - - @staticmethod - def _assert_split_model_ok(model: torch.nn.Module) -> None: - if getattr(model, "fusion_layer", None) is not None: - raise RuntimeError( - "Split ONNX export requires LiDAR-only path (fusion_layer must be None). " - "Use single-file export or implement a fusion ONNX branch." - ) - if getattr(model, "img_backbone", None) is not None: - raise RuntimeError("Split ONNX export is for LiDAR-only BEVFusion (img_backbone must be None).") - if getattr(model, "pts_middle_encoder", None) is None: - raise RuntimeError("pts_middle_encoder is required for split sparse export.") - - def _voxelize(self, model: torch.nn.Module, points: torch.Tensor) -> tuple: - """Run voxelization on a point cloud sample.""" - device = next(model.parameters()).device - points = points.to(device).float() - - with torch.no_grad(): - ret = model.pts_voxel_layer(points) - if len(ret) == 3: - feats, coords, sizes = ret - else: - feats, coords = ret - sizes = torch.ones(feats.shape[0], device=device) - from deployment.projects.bevfusion.io.coors_contract import voxel_indices_xyz_to_graph_input_zyx - - coords = coords[:, :].to(dtype=torch.int32) # [M, 3] (x, y, z) from voxel layer - coords = voxel_indices_xyz_to_graph_input_zyx(coords) # ONNX graph input: [z, y, x] - - return feats, coords, sizes - - def _get_onnx_config(self, config: BaseDeploymentConfig, component_name: str) -> Dict[str, Any]: - """Build ONNX export configuration for a components_cfg entry.""" - component_cfg = config.components_cfg.get_component(component_name) - io_cfg = component_cfg.io - - input_names = [inp.name for inp in io_cfg.inputs] - output_names = [out.name for out in io_cfg.outputs] - - dynamic_axes = {} - if hasattr(io_cfg, "dynamic_axes") and io_cfg.dynamic_axes: - dynamic_axes = dict(io_cfg.dynamic_axes) - - onnx_settings = config.deploy_cfg.get("onnx_config", {}) or {} - opset_version = getattr(onnx_settings, "opset_version", 17) - # Default "auto" = trace on the same device as the model (usually CUDA). CUDA-built spconv - # implicit_gemm runs GPU kernels: indices/features on CPU + those kernels => cudaErrorIllegalAddress. - # If you lack GPU memory for dense(), set trace_device=cpu only with a CPU spconv build, or export - # on another machine. - trace_device = getattr(onnx_settings, "trace_device", None) or os.environ.get( - "BEVFUSION_ONNX_TRACE_DEVICE", "auto" - ) - - return { - "input_names": input_names, - "output_names": output_names, - "dynamic_axes": dynamic_axes, - "opset_version": opset_version, - "do_constant_folding": bool(getattr(onnx_settings, "do_constant_folding", True)), - "export_params": True, - "keep_initializers_as_inputs": False, - "verbose": False, - "trace_device": trace_device, - } - - def _torch_onnx_export_module( - self, - module: nn.Module, - model_inputs: tuple, - output_path: str, - onnx_cfg: Dict[str, Any], - ) -> None: - """Run ``torch.onnx.export`` with deploy ``onnx_config`` (incl. ``do_constant_folding``).""" - export_kw: Dict[str, Any] = dict( - export_params=onnx_cfg["export_params"], - input_names=onnx_cfg["input_names"], - output_names=onnx_cfg["output_names"], - opset_version=onnx_cfg["opset_version"], - dynamic_axes=onnx_cfg["dynamic_axes"], - keep_initializers_as_inputs=onnx_cfg["keep_initializers_as_inputs"], - verbose=onnx_cfg["verbose"], - do_constant_folding=bool(onnx_cfg.get("do_constant_folding", True)), - ) - try: - from torch.onnx import TrainingMode - - export_kw["training"] = TrainingMode.EVAL - except Exception: - pass - - with torch.no_grad(): - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message=".*non-tuple sequence for multidimensional indexing.*", - category=UserWarning, - ) - torch.onnx.export(module, model_inputs, output_path, **export_kw) - - def _resolve_trace_device( - self, - model_device: torch.device, - onnx_cfg: Dict[str, Any], - *, - warn_on_cpu: bool = False, - ) -> torch.device: - """Resolve the tracing device from ``onnx_cfg``, coercing CPU back to the model's GPU. - - CUDA-built spconv implicit_gemm cannot run with CPU indices (merge_sort illegal - address), so a requested ``cpu`` trace device is overridden to the model's CUDA device. - """ - raw_td = onnx_cfg.get("trace_device") or "auto" - trace_dev = model_device if raw_td in ("auto", "", None) else torch.device(raw_td) - - if model_device.type == "cuda" and trace_dev.type == "cpu": - if warn_on_cpu: - self.logger.warning( - "trace_device=cpu while model is on %s: CUDA spconv implicit_gemm does not support " - "CPU indices (merge_sort illegal address). Tracing on %s instead. " - "For large dense() OOM, use a larger GPU or export elsewhere; do not use CPU trace with CUDA spconv.", - model_device, - model_device, - ) - trace_dev = model_device - return trace_dev - - @contextlib.contextmanager - def _model_on_trace_device( - self, - model: torch.nn.Module, - model_device: torch.device, - trace_dev: torch.device, - ): - """Temporarily move ``model`` to ``trace_dev`` for tracing, restoring it afterward.""" - moved = trace_dev != model_device - if moved: - self.logger.info( - "Moving model to %s for ONNX tracing (model was on %s; avoids GPU OOM from sparse dense()).", - trace_dev, - model_device, - ) - model.to(trace_dev) - try: - yield - finally: - if moved: - try: - model.to(model_device) - except Exception as e: - self.logger.warning( - "Could not move model back to %s after ONNX export (GPU may be in error state): %s", - model_device, - e, - ) - if model_device.type == "cuda": - try: - torch.cuda.empty_cache() - except Exception: - pass - - def _export_dense_to_onnx( - self, - model: torch.nn.Module, - lidar_bev: torch.Tensor, - output_path: str, - onnx_cfg: Dict[str, Any], - ) -> None: - """Export pts_backbone + neck + head (+ postprocess) to ONNX.""" - model_device = next(model.parameters()).device - trace_dev = self._resolve_trace_device(model_device, onnx_cfg) - - with self._model_on_trace_device(model, model_device, trace_dev): - wrapper = BEVFusionDenseWrapper(model) - wrapper.eval() - wrapper.to(trace_dev) - bev = lidar_bev.to(trace_dev) - self._torch_onnx_export_module(wrapper, (bev,), output_path, onnx_cfg) - - self.logger.info("Exported dense ONNX to %s", output_path) - - def _export_to_onnx( - self, - model: torch.nn.Module, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - output_path: str, - onnx_cfg: Dict[str, Any], - *, - wrapper: str = "main", - fuse_spconv_bn: bool = False, - ) -> None: - """Export voxel-based subgraph to ONNX (full main_body or sparse-only).""" - model_device = next(model.parameters()).device - trace_dev = self._resolve_trace_device(model_device, onnx_cfg, warn_on_cpu=True) - - with self._model_on_trace_device(model, model_device, trace_dev): - orig_sparse_encoder: Optional[nn.Module] = None - try: - orig_sparse_encoder = self._maybe_swap_in_float_shadow_encoder( - model, trace_dev, fuse_spconv_bn=fuse_spconv_bn - ) - - if wrapper == "sparse": - wrapper_mod: nn.Module = BEVFusionSparseWrapper(model) - elif wrapper == "main": - wrapper_mod = BEVFusionMainBodyWrapper(model) - else: - raise ValueError(f"Unknown wrapper '{wrapper}' for ONNX export") - - model_inputs = ( - voxels.to(trace_dev), - coors.to(device=trace_dev, dtype=torch.int32), - num_points_per_voxel.to(trace_dev), - ) - wrapper_mod.eval() - wrapper_mod.to(trace_dev) - - self._torch_onnx_export_module(wrapper_mod, model_inputs, output_path, onnx_cfg) - finally: - if orig_sparse_encoder is not None: - model.pts_middle_encoder = orig_sparse_encoder - - self.logger.info("Exported ONNX to %s", output_path) - - def _maybe_swap_in_float_shadow_encoder( - self, - model: torch.nn.Module, - trace_dev: torch.device, - *, - fuse_spconv_bn: bool, - ) -> Optional[nn.Module]: - """Swap ``pts_middle_encoder`` for a fused FP32 shadow used only during tracing. - - Returns the original encoder (to restore after export) or ``None`` if no swap was - needed. The shadow lets BN be folded (``fuse_spconv_bn``) into a clean sparse ONNX - graph without mutating the runtime model. - """ - enc = getattr(model, "pts_middle_encoder", None) - if enc is None: - return None - - from deployment.projects.bevfusion.export.sparse_encoder_float_shadow import ( - build_float_sparse_encoder_shadow, - resolve_sparse_onnx_shadow, - ) - - gm_src, cfg_ov = resolve_sparse_onnx_shadow(enc, model) - if gm_src is None: - return None - - self.logger.info( - "Sparse tower: using fused FP32 shadow encoder for ONNX export " - "(weights copied from source sparse encoder)." - ) - if cfg_ov: - self.logger.info( - "Shadow rebuild merges %d attribute(s) from model.cfg pts_middle_encoder.", - len(cfg_ov), - ) - - model.pts_middle_encoder = build_float_sparse_encoder_shadow( - gm_src, - trace_dev, - cfg_overrides=cfg_ov if cfg_ov else None, - fuse_spconv_bn=bool(fuse_spconv_bn), - ) - return enc - - def _get_num_proposals(self, model: torch.nn.Module) -> int: - """Extract num_proposals from the BEVFusion model config.""" - cfg = getattr(model, "cfg", None) - if cfg is not None: - num_proposals = cfg.get("num_proposals", None) - if num_proposals is not None: - return int(num_proposals) - - if hasattr(model, "bbox_head") and hasattr(model.bbox_head, "num_proposals"): - return int(model.bbox_head.num_proposals) - - raise ValueError( - "num_proposals not found in model config or bbox_head. " - "Ensure model_cfg or bbox_head.num_proposals is set." - ) - - def _fix_topk(self, input_path: str, output_path: str, num_proposals: int) -> None: - """Fix the TopK node in the ONNX graph to use a constant K. - - TensorRT requires TopK's K to be a constant, but torch.onnx.export - may produce a dynamic K. This replaces it with num_proposals. - """ - self.logger.info(f"Fixing TopK (K={num_proposals}) in ONNX graph...") - model = onnx.load(input_path) - graph = gs.import_onnx(model) - - topk_nodes = [node for node in graph.nodes if node.op == "TopK"] - if len(topk_nodes) == 0: - self.logger.warning("No TopK node found; skipping fix") - onnx.save_model(model, output_path) - return - - if len(topk_nodes) != 1: - self.logger.warning(f"Expected 1 TopK node, found {len(topk_nodes)}; fixing the first one") - - topk = topk_nodes[0] - topk.inputs[1] = gs.Constant("K", values=np.array([num_proposals], dtype=np.int64)) - topk.outputs[0].shape = [1, num_proposals] - topk.outputs[0].dtype = topk.inputs[0].dtype if topk.inputs[0].dtype else np.float32 - topk.outputs[1].shape = [1, num_proposals] - topk.outputs[1].dtype = np.int64 - - graph.cleanup().toposort() - onnx.save_model(gs.export_onnx(graph), output_path) - - # Clean up temp file - if os.path.exists(input_path) and input_path != output_path: - os.remove(input_path) - - self.logger.info(f"TopK fixed. Final ONNX: {output_path}") diff --git a/deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py b/deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py deleted file mode 100644 index 207db2088..000000000 --- a/deployment/projects/bevfusion/export/onnx_fuse_implicit_gemm_activation.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Fuse post-spconv activation into ``autoware`` ImplicitGemm plugin ``act_type``. - -TensorRT does not fuse standard ONNX ``Relu`` with custom ops, so we fold -``ImplicitGemm → Relu`` by setting ``act_type=kReLU`` on the producer node and -removing the standalone ``Relu`` node. -""" - -from __future__ import annotations - -from collections import defaultdict -from typing import DefaultDict, Dict, List, Optional, Set - -import numpy as np -import onnx -from onnx import helper, numpy_helper - - -def _normalize_attr(name: str) -> str: - """Strip ONNX type suffix (``_f``, ``_i``, ``_s``, ``_l``) from an attribute name.""" - for suf in ("_f", "_i", "_s", "_l"): - if name.endswith(suf) and len(name) > len(suf): - return name[: -len(suf)] - return name - - -def _try_get_constant_numpy( - graph: onnx.GraphProto, - name: str, - init_map: Dict[str, np.ndarray], -) -> Optional[np.ndarray]: - """Return the constant numpy array for tensor ``name``, or ``None`` if not a constant. - - Checks ``init_map`` (pre-built from ``graph.initializer``) first, then searches - for a ``Constant`` node that produces ``name``. - """ - if name in init_map: - return init_map[name] - for node in graph.node: - if node.op_type != "Constant": - continue - if name in node.output: - for attr in node.attribute: - if attr.type == onnx.AttributeProto.TENSOR: - return numpy_helper.to_array(attr.t) - return None - - -def _read_implicit_gemm_attrs(node: onnx.NodeProto) -> Dict[str, object]: - out: Dict[str, object] = {} - for attr in node.attribute: - base = _normalize_attr(attr.name) - if attr.type == onnx.AttributeProto.INT: - out[base] = int(attr.i) - elif attr.type == onnx.AttributeProto.FLOAT: - out[base] = float(attr.f) - return out - - -def _replace_tensor_name(graph: onnx.GraphProto, old: str, new: str) -> None: - if old == new: - return - for n in graph.node: - for i, inp in enumerate(n.input): - if inp == old: - n.input[i] = new - for out in graph.output: - if out.name == old: - out.name = new - for vi in graph.value_info: - if vi.name == old: - vi.name = new - - -def _set_implicit_gemm_act_type(node: onnx.NodeProto, act_type: int) -> None: - kept = [a for a in node.attribute if _normalize_attr(a.name) != "act_type"] - del node.attribute[:] - node.attribute.extend(kept) - node.attribute.append(helper.make_attribute("act_type", int(act_type))) - - -def fuse_autoware_implicit_gemm_trailing_relu(model: onnx.ModelProto) -> int: - """Set ``act_type`` = kReLU (1) on ImplicitGemm and remove redundant ``Relu`` nodes.""" - - graph = model.graph - n_removed = 0 - - remove_idx: Set[int] = set() - - for ri, relu in enumerate(graph.node): - if ri in remove_idx: - continue - if relu.op_type != "Relu": - continue - if relu.domain not in ("", "ai.onnx"): - continue - if len(relu.input) < 1 or not relu.input[0]: - continue - if len(relu.output) < 1 or not relu.output[0]: - continue - - users: DefaultDict[str, List[int]] = defaultdict(list) - for ni, n in enumerate(graph.node): - if ni in remove_idx: - continue - for inp in n.input: - if inp: - users[inp].append(ni) - - g_out = relu.input[0] - r_out = relu.output[0] - - if len(users.get(g_out, [])) != 1: - continue - - producer_i: int | None = None - producer: onnx.NodeProto | None = None - for ni, n in enumerate(graph.node): - if g_out in n.output: - producer_i = ni - producer = n - break - if producer is None or producer_i is None: - continue - if producer.op_type != "ImplicitGemm" or producer.domain != "autoware": - continue - if producer_i in remove_idx: - continue - - attrs = _read_implicit_gemm_attrs(producer) - cur = int(attrs.get("act_type", 0) or 0) - if cur not in (0, 1): - continue - - _set_implicit_gemm_act_type(producer, 1) - _replace_tensor_name(graph, r_out, g_out) - remove_idx.add(ri) - n_removed += 1 - - if not remove_idx: - return 0 - - new_nodes: List[onnx.NodeProto] = [] - for ni, n in enumerate(graph.node): - if ni not in remove_idx: - new_nodes.append(n) - del graph.node[:] - graph.node.extend(new_nodes) - return n_removed diff --git a/deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py b/deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py deleted file mode 100644 index a3584b43b..000000000 --- a/deployment/projects/bevfusion/export/sparse_encoder_float_shadow.py +++ /dev/null @@ -1,295 +0,0 @@ -"""FP32 sparse-encoder shadow for ``torch.onnx.export``. - -For split export the sparse tower is traced separately. This helper rebuilds a -fused FP32 ``BEVFusionSparseEncoder`` and copies float weights from the source -encoder, so BN can be folded (``fuse_spconv_bn``) into a clean sparse ONNX graph -without mutating the runtime model. -""" - -from __future__ import annotations - -import copy -import logging -from typing import Any, Dict, Mapping, Optional, Tuple - -import torch -import torch.nn as nn - -logger = logging.getLogger(__name__) - -# Attributes required to rebuild FP32 shadow encoder. -SPARSE_ENCODER_SHADOW_ATTRS: tuple[str, ...] = ( - "sparse_shape", - "in_channels", - "base_channels", - "output_channels", - "encoder_channels", - "encoder_paddings", - "num_aug_features", - "aug_features_min_values", - "aug_features_max_values", -) - - -def has_sparse_encoder_shadow_attributes(module: nn.Module) -> bool: - """True if ``module`` carries the config fields needed by ``build_float_sparse_encoder_shadow``.""" - return all(hasattr(module, name) for name in SPARSE_ENCODER_SHADOW_ATTRS) - - -def encoder_cfg_overrides_from_bevfusion_model(model: Optional[nn.Module]) -> Dict[str, Any]: - """Build shadow-attribute overrides from ``model.cfg.model['pts_middle_encoder']`` (MMEngine config).""" - if model is None: - return {} - cfg = getattr(model, "cfg", None) - if cfg is None: - return {} - model_dict = getattr(cfg, "model", None) - if model_dict is None: - return {} - try: - enc = model_dict.get("pts_middle_encoder") if hasattr(model_dict, "get") else None - except Exception: - return {} - if enc is None: - return {} - if isinstance(enc, Mapping) and not isinstance(enc, dict): - try: - enc = dict(enc) - except Exception: - return {} - if not isinstance(enc, dict): - return {} - out: Dict[str, Any] = {} - for name in SPARSE_ENCODER_SHADOW_ATTRS: - if name in enc: - out[name] = copy.deepcopy(enc[name]) - for name in ("norm_cfg", "block_type", "order", "return_middle_feats"): - if name in enc: - out[name] = copy.deepcopy(enc[name]) - return out - - -def resolve_sparse_onnx_shadow( - pts_middle_encoder: Optional[nn.Module], - bevfusion: Optional[nn.Module] = None, -) -> Tuple[Optional[nn.Module], Dict[str, Any]]: - """Pick sparse encoder source module and optional config overrides for shadow rebuild.""" - if pts_middle_encoder is None: - return None, {} - overrides = encoder_cfg_overrides_from_bevfusion_model(bevfusion) - if has_sparse_encoder_shadow_attributes(pts_middle_encoder): - return pts_middle_encoder, overrides - - can_fill = all(hasattr(pts_middle_encoder, name) or (name in overrides) for name in SPARSE_ENCODER_SHADOW_ATTRS) - if can_fill: - if overrides: - logger.info( - "Sparse ONNX shadow: merging %d key(s) from model.cfg pts_middle_encoder.", - len(overrides), - ) - return pts_middle_encoder, overrides - - logger.info( - "Sparse ONNX shadow: pts_middle_encoder lacks the config fields needed to rebuild an " - "FP32 shadow and model.cfg is incomplete; tracing the encoder directly." - ) - return None, {} - - -def build_float_sparse_encoder_shadow( - gm: nn.Module, - device: torch.device, - *, - cfg_overrides: Optional[Dict[str, Any]] = None, - fuse_spconv_bn: bool = True, -) -> nn.Module: - """Construct a fused FP32 ``BEVFusionSparseEncoder`` and load weights from ``gm`` state_dict. - - Only floating conv/BN parameters are copied from ``gm``. ``cfg_overrides`` supplies - fields missing on the source module (from ``model.cfg``). - """ - from mmengine.registry import MODELS, init_default_scope - - import projects.BEVFusion.bevfusion # noqa: F401 — register BEVFusionSparseEncoder - - init_default_scope("mmdet3d") - - def _pick(name: str) -> Any: - if cfg_overrides is not None and name in cfg_overrides: - return cfg_overrides[name] - return getattr(gm, name, None) - - missing = [r for r in SPARSE_ENCODER_SHADOW_ATTRS if _pick(r) is None] - if missing: - raise RuntimeError( - "Cannot rebuild FP32 sparse encoder for ONNX: source encoder + overrides missing: " - f"{missing}. Ensure the FP32 shadow encoder defines these (match training sparse_encoder), or pass " - f"a BEVFusion model with model.cfg.model.pts_middle_encoder." - ) - - def _buf_to_list(buf: torch.Tensor) -> list: - return buf.detach().cpu().flatten().tolist() - - def _aug_to_list(val: Any) -> list: - if isinstance(val, torch.Tensor): - return _buf_to_list(val) - if isinstance(val, (list, tuple)): - return list(val) - raise TypeError(f"aug_features_* must be tensor or list, got {type(val)!r}") - - default_norm = dict(type="BN1d", eps=1e-3, momentum=0.01) - nc = _pick("norm_cfg") - norm_cfg = copy.deepcopy(nc if nc is not None else default_norm) - - block_type = _pick("block_type") - if block_type is None: - block_type = "basicblock" - order_val = _pick("order") - order = tuple(order_val) if order_val is not None else ("conv", "norm", "act") - - enc_channels = _pick("encoder_channels") - if isinstance(enc_channels, torch.Tensor): - raise TypeError("encoder_channels must be nested tuples, not Tensor") - enc_paddings = _pick("encoder_paddings") - sparse_shape = _pick("sparse_shape") - sparse_shape = list(sparse_shape) if not isinstance(sparse_shape, list) else list(sparse_shape) - - ret_mid = _pick("return_middle_feats") - return_middle_feats = bool(ret_mid) if ret_mid is not None else False - - enc_cfg: Dict[str, Any] = dict( - type="BEVFusionSparseEncoder", - in_channels=int(_pick("in_channels")), - aug_features_min_values=_aug_to_list(_pick("aug_features_min_values")), - aug_features_max_values=_aug_to_list(_pick("aug_features_max_values")), - num_aug_features=int(_pick("num_aug_features")), - sparse_shape=sparse_shape, - order=order, - norm_cfg=norm_cfg, - base_channels=int(_pick("base_channels")), - output_channels=int(_pick("output_channels")), - encoder_channels=enc_channels, - encoder_paddings=enc_paddings, - block_type=block_type, - return_middle_feats=return_middle_feats, - ) - - enc: nn.Module = MODELS.build(enc_cfg) - enc.to(device) - enc.eval() - - if fuse_spconv_bn: - from deployment.projects.bevfusion.export.spconv_bn_fusion import fuse_spconv_bn_in_encoder - - fuse_spconv_bn_in_encoder(enc) - else: - logger.info("Sparse ONNX float shadow: keep SparseConv+BN unfused (fuse_spconv_bn=False).") - - gm_sd = gm.state_dict() - enc_sd = enc.state_dict() - - def _align_5d_spconv_weight_to_krsc(v: torch.Tensor, target: torch.Size) -> Optional[torch.Tensor]: - """Some checkpoints store 5D sparse conv as (C_in, C_out, Kz, Ky, Kx); MMDet encoder uses KRSC (C_out, Kz, Ky, Kx, C_in).""" - if v.dim() != 5 or len(target) != 5: - return None - if v.shape == target: - return v - # Explicit ICOC -> KRSC when channel/spatial layout matches. - if ( - v.shape[0] == target[4] - and v.shape[1] == target[0] - and v.shape[2] == target[1] - and v.shape[3] == target[2] - and v.shape[4] == target[3] - ): - return v.permute(1, 2, 3, 4, 0).contiguous() - perm = v.permute(1, 2, 3, 4, 0).contiguous() - if perm.shape == target: - return perm - perm2 = v.permute(4, 0, 1, 2, 3).contiguous() - if perm2.shape == target: - return perm2 - return None - - def _flat_state_key(key: str) -> str: - """Legacy checkpoints may use underscore keys (e.g. ``encoder_layers_encoder_layer1_0_conv1``).""" - return key.replace(".", "_") - - def _gm_value_for_key(key: str) -> Optional[torch.Tensor]: - flat = _flat_state_key(key) - for cand in ( - key, - f"module.{key}", - f"pts_middle_encoder.{key}", - flat, - f"module.{flat}", - f"pts_middle_encoder.{flat}", - ): - if cand in gm_sd: - return gm_sd[cand] # type: ignore[return-value] - if key.startswith("module.") and key[len("module.") :] in gm_sd: - return gm_sd[key[len("module.") :]] # type: ignore[return-value] - return None - - # Copy with ``Tensor.copy_`` instead of ``load_state_dict``: spconv registers - # ``load_state_dict`` pre-hooks that permute *disk* layouts when SPCONV_SAVED_WEIGHT_LAYOUT - # is set; mutating the same dict we validated can also desync shapes vs. plain Parameters. - n_copied = 0 - with torch.no_grad(): - for k, t in enc_sd.items(): - v = _gm_value_for_key(k) - if v is None or not torch.is_tensor(v): - continue - - v = v.detach() - if v.dim() == 5 and t.dim() == 5 and v.shape != t.shape: - aligned = _align_5d_spconv_weight_to_krsc(v, t.shape) - if aligned is not None: - logger.info( - "Float shadow ICOC->KRSC %s: %s -> %s", - k, - tuple(v.shape), - tuple(aligned.shape), - ) - v = aligned - else: - logger.debug("Float shadow skip 5D %s: gm %s vs enc %s", k, tuple(v.shape), tuple(t.shape)) - continue - elif v.shape != t.shape: - continue - - if v.dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64): - w = v.to(device=t.device, dtype=t.dtype, non_blocking=False).contiguous() - elif v.dtype in (torch.int32, torch.int64, torch.bool): - w = v.to(device=t.device, non_blocking=False).contiguous() - else: - continue - - if w.shape != t.shape: - raise RuntimeError( - f"Float shadow internal error: key {k!r} tensor shape {tuple(w.shape)} vs encoder " - f"{tuple(t.shape)} after layout fix." - ) - - parent_path, dot, leaf = k.rpartition(".") - if not dot: - continue - try: - sub = enc.get_submodule(parent_path) - except AttributeError: - logger.debug("Float shadow: no submodule for state key %s", k) - continue - dst = getattr(sub, leaf, None) - if dst is None or not torch.is_tensor(dst): - continue - dst.copy_(w) - n_copied += 1 - - logger.info( - "Sparse ONNX float shadow: copied %d / %d state entries from source encoder via in-place copy " - "(bypasses spconv load_state_dict hooks).", - n_copied, - len(enc_sd), - ) - - return enc diff --git a/deployment/projects/bevfusion/export/tensorrt_export_pipeline.py b/deployment/projects/bevfusion/export/tensorrt_export_pipeline.py deleted file mode 100644 index c9d299c85..000000000 --- a/deployment/projects/bevfusion/export/tensorrt_export_pipeline.py +++ /dev/null @@ -1,136 +0,0 @@ -"""BEVFusion TensorRT export pipeline. - -Converts BEVFusion ONNX (single or split sparse+dense) to TensorRT engine(s). -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Dict, Optional, Tuple - -import torch - -from deployment.config.base import BaseDeploymentConfig -from deployment.config.schema import ComponentsConfig -from deployment.export.exporters.tensorrt_exporter import TensorRTExporter -from deployment.primitives.artifacts import Artifact -from deployment.primitives.device import DeviceSpec -from deployment.primitives.tensorrt_plugins import load_tensorrt_plugin_libraries -from deployment.projects.bevfusion.io.component_utils import is_split_bevfusion_components - - -class BEVFusionTensorRTExportPipeline: - """TensorRT export for BEVFusion (one engine or sparse+dense pair).""" - - def __init__( - self, - components_cfg: ComponentsConfig, - plugin_libraries: Tuple[str, ...] = (), - logger: Optional[logging.Logger] = None, - ) -> None: - self._components_cfg = components_cfg - self._plugin_libraries = plugin_libraries - self.logger = logger or logging.getLogger(__name__) - - def export( - self, - *, - onnx_path: str, - output_dir: str, - config: BaseDeploymentConfig, - device: DeviceSpec, - ) -> Artifact: - if not device.is_cuda: - raise ValueError(f"TensorRT export requires CUDA device, got: {device}") - - torch.cuda.set_device(device.index) - - # Load any custom plugin .so libraries (e.g. the BEVFusion spconv ImplicitGemm plugin) - # before building. No-op when plugin_libraries is empty. - load_tensorrt_plugin_libraries(self.logger, getattr(self, "_plugin_libraries", ())) - - onnx_dir = Path(onnx_path) - output_dir_path = Path(output_dir) - output_dir_path.mkdir(parents=True, exist_ok=True) - - if is_split_bevfusion_components(self._components_cfg): - return self._export_split_engines(onnx_dir, output_dir_path, config) - - component_cfg = self._components_cfg.get_component("bevfusion_main_body") - onnx_file = onnx_dir / component_cfg.onnx_file - engine_file = output_dir_path / component_cfg.engine_file - - if not onnx_file.exists(): - raise FileNotFoundError(f"ONNX file not found: {onnx_file}") - - self.logger.info("=" * 80) - self.logger.info("Converting BEVFusion ONNX to TensorRT") - self.logger.info("=" * 80) - self.logger.info(f"ONNX: {onnx_file}") - self.logger.info(f"Engine: {engine_file}") - - artifact = TensorRTExporter(config=config.get_tensorrt_settings("bevfusion_main_body")).export( - onnx_path=str(onnx_file), - output_path=str(engine_file), - ) - - self.logger.info(f"TensorRT engine saved: {artifact.path}") - self.logger.info("=" * 80) - - return Artifact(path=str(output_dir_path)) - - def _export_split_engines( - self, - onnx_dir: Path, - output_dir_path: Path, - config: BaseDeploymentConfig, - ) -> Artifact: - if not onnx_dir.is_dir(): - raise ValueError(f"Split TensorRT export expects ONNX directory, got: {onnx_dir}") - - onnx_files = sorted( - (path for path in onnx_dir.iterdir() if path.is_file() and path.suffix.lower() == ".onnx"), - key=lambda p: p.name, - ) - if not onnx_files: - raise FileNotFoundError(f"No ONNX files in {onnx_dir}") - - engine_file_map = self._build_engine_file_map() - onnx_stem_to_component = self._build_onnx_stem_to_component_map() - - self.logger.info("=" * 80) - self.logger.info("Converting split BEVFusion ONNX → TensorRT (sparse + dense)") - self.logger.info("=" * 80) - - for i, onnx_file in enumerate(onnx_files, 1): - onnx_stem = onnx_file.stem - if onnx_stem not in engine_file_map: - raise KeyError(f"ONNX file '{onnx_file.name}' is not declared in deploy config components.*.onnx_file") - engine_file = engine_file_map[onnx_stem] - trt_path = output_dir_path / engine_file - trt_path.parent.mkdir(parents=True, exist_ok=True) - - component_name = onnx_stem_to_component[onnx_stem] - self.logger.info("[%d/%d] %s → %s", i, len(onnx_files), onnx_file.name, trt_path.name) - - TensorRTExporter(config=config.get_tensorrt_settings(component_name)).export( - onnx_path=str(onnx_file), - output_path=str(trt_path), - ) - - self.logger.info("Split TensorRT engines written to %s", output_dir_path) - self.logger.info("=" * 80) - return Artifact(path=str(output_dir_path)) - - def _build_engine_file_map(self) -> Dict[str, str]: - mapping: Dict[str, str] = {} - for _name, comp in self._components_cfg.items(): - mapping[Path(comp.onnx_file).stem] = comp.engine_file - return mapping - - def _build_onnx_stem_to_component_map(self) -> Dict[str, str]: - return { - Path(component_cfg.onnx_file).stem: component_name - for component_name, component_cfg in self._components_cfg.items() - } diff --git a/deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py b/deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py deleted file mode 100644 index b029ac940..000000000 --- a/deployment/projects/bevfusion/inference/bevfusion_inference_pipeline.py +++ /dev/null @@ -1,310 +0,0 @@ -"""BEVFusion Deployment Pipeline Base Class. - -Provides common preprocessing, postprocessing, and inference logic -shared by PyTorch, ONNX, and TensorRT backend implementations. -""" - -from __future__ import annotations - -import logging -import os -import time -from abc import abstractmethod -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -import torch -from typing_extensions import override - -from deployment.config.enums import Backend -from deployment.inference.base_inference_pipeline import BaseInferencePipeline -from deployment.primitives.device import DeviceSpec - -logger = logging.getLogger(__name__) - -_DEBUG_POSTPROCESS = os.environ.get("BEVFUSION_DEBUG_POSTPROCESS", "").strip().lower() in ("1", "true", "yes") -_DEBUG_POSTPROCESS_MAX = 2 -_postprocess_debug_count = 0 - - -def _env_int_pp(key: str, default: int) -> int: - try: - return int(os.environ.get(key, str(default)).strip()) - except ValueError: - return default - - -class BEVFusionDeploymentPipeline(BaseInferencePipeline): - """Base pipeline for BEVFusion inference. - - Handles voxelization in preprocessing and bbox decoding in postprocessing. - The model (ONNX/TensorRT) takes voxels/coors/num_points_per_voxel and - outputs bbox_pred/score/label_pred directly. - """ - - def __init__( - self, - pytorch_model: torch.nn.Module, - backend_type: Backend, - device: DeviceSpec, - ) -> None: - cfg = getattr(pytorch_model, "cfg", None) - - class_names = getattr(cfg, "class_names", None) - point_cloud_range = getattr(cfg, "point_cloud_range", None) - voxel_size = getattr(cfg, "voxel_size", None) - - if class_names is None: - raise ValueError("class_names not found in pytorch_model.cfg") - - super().__init__( - model=pytorch_model, - backend_type=backend_type, - device=device, - ) - - self.pytorch_model: torch.nn.Module = pytorch_model - self.num_classes: int = len(class_names) - self.class_names: List[str] = class_names - self.point_cloud_range: Optional[List[float]] = point_cloud_range - self.voxel_size: Optional[List[float]] = voxel_size - - def to_device_tensor(self, data: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: - if isinstance(data, np.ndarray): - data = torch.from_numpy(data) - return data.to(self.torch_device) - - def to_numpy(self, data: torch.Tensor, dtype: np.dtype = np.float32) -> np.ndarray: - arr = data.cpu().numpy().astype(dtype) - if not arr.flags["C_CONTIGUOUS"]: - arr = np.ascontiguousarray(arr) - return arr - - @override - def preprocess( - self, - points: torch.Tensor, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, object]]: - """Voxelize point cloud into voxels/coors/num_points_per_voxel. - - Uses the BEVFusion model's voxelization layer (outside the ONNX graph). - - Args: - points: Point cloud tensor [N, point_features]. - - Returns: - Tuple of (preprocessed_dict, metadata_dict). - """ - points_tensor = self.to_device_tensor(points).float() - - with torch.no_grad(): - feats, coords, sizes = [], [], [] - ret = self.pytorch_model.pts_voxel_layer(points_tensor) - if len(ret) == 3: - f, c, n = ret - else: - f, c = ret - n = None - feats.append(f) - coords.append(c) - if n is not None: - sizes.append(n) - - voxels = torch.cat(feats, dim=0) - coors = torch.cat(coords, dim=0) - num_points_per_voxel = ( - torch.cat(sizes, dim=0) if sizes else torch.ones(voxels.shape[0], device=voxels.device) - ) - - preprocessed_dict = { - "voxels": voxels, - "coors": coors, - "num_points_per_voxel": num_points_per_voxel, - } - return preprocessed_dict, {} - - @override - def run_model( - self, - preprocessed_input: Dict[str, torch.Tensor], - ) -> Tuple[List[torch.Tensor], Dict[str, float]]: - """Run the BEVFusion model and return raw outputs with latency. - - Args: - preprocessed_input: Dict with voxels, coors, num_points_per_voxel. - - Returns: - Tuple of ([bbox_pred, score, label_pred], stage_latencies). - """ - stage_latencies: Dict[str, float] = {} - - start = time.perf_counter() - outputs = self.run_bevfusion( - preprocessed_input["voxels"], - preprocessed_input["coors"], - preprocessed_input["num_points_per_voxel"], - ) - stage_latencies["bevfusion_ms"] = (time.perf_counter() - start) * 1000 - - return outputs, stage_latencies - - @override - def postprocess( - self, - model_outputs: List[torch.Tensor], - sample_meta: Dict[str, object], - ) -> List[Dict[str, Union[List[float], float, int]]]: - """Decode bbox_pred/score/label_pred into detection dicts. - - The BEVFusion ONNX model already includes query scoring / selection, but - bbox outputs are still in the head-encoded space and must be decoded to - metric coordinates: - - bbox_pred: [10, num_proposals] - (center_x_feat, center_y_feat, z_gravity, dim0_log, dim1_log, dim2_log, sin, cos, vx, vy) - - score: [num_proposals] - - label_pred: [num_proposals] - - Args: - model_outputs: [bbox_pred, score, label_pred] tensors. - sample_meta: Sample metadata. - - Returns: - List of detection dicts with bbox_3d, score, label. - """ - bbox_pred, score, label_pred = [self.to_device_tensor(o) for o in model_outputs] - - # Normalize common export/runtime shapes to [10, num_proposals], [num_proposals], [num_proposals]. - if bbox_pred.ndim == 3 and bbox_pred.shape[0] == 1: - bbox_pred = bbox_pred[0] - if bbox_pred.ndim == 2 and bbox_pred.shape[0] != 10 and bbox_pred.shape[1] == 10: - bbox_pred = bbox_pred.transpose(0, 1).contiguous() - if bbox_pred.ndim != 2 or bbox_pred.shape[0] != 10: - logger.warning(f"Unexpected bbox_pred shape {tuple(bbox_pred.shape)}; skipping frame.") - return [] - - score = score.reshape(-1) - label_pred = label_pred.reshape(-1) - - num_proposals = min(bbox_pred.shape[1], score.shape[0], label_pred.shape[0]) - if num_proposals == 0: - return [] - - global _postprocess_debug_count - dbg_max = max(0, _env_int_pp("BEVFUSION_DEBUG_POSTPROCESS_FRAMES", _DEBUG_POSTPROCESS_MAX)) - did_pp_dbg = False - if _DEBUG_POSTPROCESS and _postprocess_debug_count < dbg_max: - _postprocess_debug_count += 1 - did_pp_dbg = True - sc = score[:num_proposals].float() - lb = label_pred[:num_proposals].long() - cx = bbox_pred[0, :num_proposals].float() - cy = bbox_pred[1, :num_proposals].float() - uniq_l = torch.unique(lb) - logger.warning( - "[debug-postprocess] frame=%d/%d backend=%s num_proposals=%d " - "score[min,max,mean]=[%.6f,%.6f,%.6f] score>0.1:%d score>0.5:%d " - "label[min,max]=[%d,%d] label_unique=%s " - "center_feat_x[min,max]=[%.4f,%.4f] center_feat_y[min,max]=[%.4f,%.4f]", - _postprocess_debug_count, - dbg_max, - str(self.backend_type), - int(num_proposals), - float(sc.min().item()), - float(sc.max().item()), - float(sc.mean().item()), - int((sc > 0.1).sum().item()), - int((sc > 0.5).sum().item()), - int(lb.min().item()), - int(lb.max().item()), - str(uniq_l.detach().cpu().tolist()), - float(cx.min().item()), - float(cx.max().item()), - float(cy.min().item()), - float(cy.max().item()), - ) - - # Decode via BEVFusion's own bbox_coder to avoid convention drift. - bbox_coder = getattr(self.pytorch_model.bbox_head, "bbox_coder", None) - if bbox_coder is None: - logger.warning("bbox_coder not found on model.bbox_head; skipping frame.") - return [] - - center = bbox_pred[0:2, :num_proposals].unsqueeze(0) - height = bbox_pred[2:3, :num_proposals].unsqueeze(0) - dim = bbox_pred[3:6, :num_proposals].unsqueeze(0) - rot = bbox_pred[6:8, :num_proposals].unsqueeze(0) - vel = bbox_pred[8:10, :num_proposals].unsqueeze(0) - - labels = label_pred[:num_proposals].long() - scores = score[:num_proposals].to(dtype=bbox_pred.dtype) - heatmap = torch.zeros((1, self.num_classes, num_proposals), device=self.torch_device, dtype=bbox_pred.dtype) - valid = (labels >= 0) & (labels < self.num_classes) - if valid.any(): - valid_idx = torch.nonzero(valid, as_tuple=False).reshape(-1) - heatmap[0, labels[valid_idx], valid_idx] = scores[valid_idx] - - decoded = bbox_coder.decode(heatmap, rot, dim, center, height, vel, filter=False)[0] - decoded_boxes = decoded["bboxes"] - decoded_scores = decoded["scores"] - decoded_labels = decoded["labels"] - - results: List[Dict[str, Union[List[float], float, int]]] = [] - for i in range(decoded_boxes.shape[0]): - s = float(decoded_scores[i].item()) - if s < 1e-6: - continue - - bbox = decoded_boxes[i].detach().cpu().numpy() - # decoded box format: [x, y, z, dx, dy, dz, yaw, vx, vy] - if bbox.shape[0] < 7: - continue - - cx, cy, z = float(bbox[0]), float(bbox[1]), float(bbox[2]) - d0, d1, d2 = float(bbox[3]), float(bbox[4]), float(bbox[5]) - yaw = float(bbox[6]) - vx = float(bbox[7]) if bbox.shape[0] > 7 else 0.0 - vy = float(bbox[8]) if bbox.shape[0] > 8 else 0.0 - - results.append( - { - "bbox_3d": [cx, cy, z, d0, d1, d2, yaw, vx, vy], - "score": s, - "label": int(decoded_labels[i].item()), - } - ) - - if did_pp_dbg and decoded_boxes.shape[0] > 0: - b = decoded_boxes[:, :3].detach().float() - logger.warning( - "[debug-postprocess] decoded metric centers N=%d " - "x[min,max]=[%.2f,%.2f] y[min,max]=[%.2f,%.2f] z[min,max]=[%.2f,%.2f] " - "(compare to point_cloud_range / GT — wild ranges → mAP 0 with many preds)", - int(decoded_boxes.shape[0]), - float(b[:, 0].min().item()), - float(b[:, 0].max().item()), - float(b[:, 1].min().item()), - float(b[:, 1].max().item()), - float(b[:, 2].min().item()), - float(b[:, 2].max().item()), - ) - - return results - - @abstractmethod - def run_bevfusion( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - ) -> List[torch.Tensor]: - """Run the BEVFusion model. - - Args: - voxels: [M, max_points, C] - coors: [M, 3] (z, y, x) - num_points_per_voxel: [M] - - Returns: - [bbox_pred, score, label_pred] - """ - raise NotImplementedError diff --git a/deployment/projects/bevfusion/inference/onnx_inference_pipeline.py b/deployment/projects/bevfusion/inference/onnx_inference_pipeline.py deleted file mode 100644 index 99d184200..000000000 --- a/deployment/projects/bevfusion/inference/onnx_inference_pipeline.py +++ /dev/null @@ -1,165 +0,0 @@ -"""BEVFusion ONNX Pipeline Implementation.""" - -from __future__ import annotations - -import logging -import os.path as osp -from typing import List, Optional - -import numpy as np -import onnxruntime as ort -import torch -from typing_extensions import override - -from deployment.config.enums import Backend -from deployment.config.schema import ComponentsConfig -from deployment.primitives.artifacts import resolve_artifact_path -from deployment.primitives.device import DeviceSpec -from deployment.projects.bevfusion.inference.bevfusion_inference_pipeline import BEVFusionDeploymentPipeline -from deployment.projects.bevfusion.io.component_utils import has_component, is_split_bevfusion_components -from deployment.projects.bevfusion.io.coors_contract import voxel_indices_xyz_to_graph_input_zyx - -logger = logging.getLogger(__name__) - - -class BEVFusionONNXPipeline(BEVFusionDeploymentPipeline): - """ONNXRuntime-based BEVFusion pipeline. - - Single ONNX: voxels/coors/num_points → bbox_pred/score/label_pred. - - Split ONNX: sparse session → ``lidar_bev``, then dense session → outputs. - """ - - def __init__( - self, - pytorch_model: torch.nn.Module, - onnx_dir: str, - device: DeviceSpec, - components_cfg: ComponentsConfig, - ) -> None: - super().__init__(pytorch_model=pytorch_model, backend_type=Backend.ONNX, device=device) - - self.onnx_dir = onnx_dir - self._components_cfg = components_cfg - split_layout = is_split_bevfusion_components(components_cfg) - merged_model_available = False - if split_layout and has_component(components_cfg, "bevfusion_main_body"): - merged_path = resolve_artifact_path( - base_dir=self.onnx_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_main_body", - file_key="onnx_file", - ) - merged_model_available = osp.exists(merged_path) - self._split = split_layout and not merged_model_available - self.session: Optional[ort.InferenceSession] = None - self._session_sparse: Optional[ort.InferenceSession] = None - self._session_dense: Optional[ort.InferenceSession] = None - self._load_onnx_model() - logger.info(f"BEVFusion ONNX pipeline initialized from: {onnx_dir} (split={self._split})") - - def _load_onnx_model(self) -> None: - so = ort.SessionOptions() - so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL - providers = self.device.to_ort_provider() - - if self._split: - sparse_path = resolve_artifact_path( - base_dir=self.onnx_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_sparse", - file_key="onnx_file", - ) - dense_path = resolve_artifact_path( - base_dir=self.onnx_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_dense", - file_key="onnx_file", - ) - if not osp.exists(sparse_path): - raise FileNotFoundError(f"Sparse ONNX not found: {sparse_path}") - if not osp.exists(dense_path): - raise FileNotFoundError(f"Dense ONNX not found: {dense_path}") - self._session_sparse = ort.InferenceSession(sparse_path, sess_options=so, providers=providers) - self._session_dense = ort.InferenceSession(dense_path, sess_options=so, providers=providers) - logger.info("Loaded split ONNX: %s , %s", sparse_path, dense_path) - return - - model_path = resolve_artifact_path( - base_dir=self.onnx_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_main_body", - file_key="onnx_file", - ) - if not osp.exists(model_path): - raise FileNotFoundError(f"BEVFusion ONNX not found: {model_path}") - - self.session = ort.InferenceSession(model_path, sess_options=so, providers=providers) - logger.info(f"Loaded BEVFusion ONNX: {model_path}") - - @override - def run_bevfusion( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - ) -> List[torch.Tensor]: - if self._split: - return self._run_bevfusion_split(voxels, coors, num_points_per_voxel) - - assert self.session is not None - voxels_np = self.to_numpy(voxels, dtype=np.float32) - coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) - num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) - - input_names = [inp.name for inp in self.session.get_inputs()] - output_names = [out.name for out in self.session.get_outputs()] - - feed_dict = {} - for name in input_names: - if "voxel" in name.lower() and "num" not in name.lower(): - feed_dict[name] = voxels_np - elif "coor" in name.lower(): - feed_dict[name] = coors_np - elif "num" in name.lower(): - feed_dict[name] = num_points_np - - outputs = self.session.run(output_names, feed_dict) - return [torch.from_numpy(out).to(self.torch_device) for out in outputs] - - def _run_bevfusion_split( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - ) -> List[torch.Tensor]: - assert self._session_sparse is not None and self._session_dense is not None - - voxels_np = self.to_numpy(voxels, dtype=np.float32) - coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) - num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) - - s_in = [inp.name for inp in self._session_sparse.get_inputs()] - s_out = [out.name for out in self._session_sparse.get_outputs()] - - sparse_feed = {} - for name in s_in: - ln = name.lower() - if "voxel" in ln and "num" not in ln: - sparse_feed[name] = voxels_np - elif "coor" in ln: - sparse_feed[name] = coors_np - elif "num" in ln: - sparse_feed[name] = num_points_np - - sparse_ort_outs = self._session_sparse.run(s_out, sparse_feed) - if len(sparse_ort_outs) != 1: - raise RuntimeError(f"Expected 1 sparse output, got {len(sparse_ort_outs)}") - lidar_bev_np = np.ascontiguousarray(sparse_ort_outs[0].astype(np.float32)) - - d_in = [inp.name for inp in self._session_dense.get_inputs()] - d_out = [out.name for out in self._session_dense.get_outputs()] - dense_feed = {d_in[0]: lidar_bev_np} - - dense_ort_outs = self._session_dense.run(d_out, dense_feed) - return [torch.from_numpy(out).to(self.torch_device) for out in dense_ort_outs] diff --git a/deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py b/deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py deleted file mode 100644 index 7827b05cd..000000000 --- a/deployment/projects/bevfusion/inference/pytorch_inference_pipeline.py +++ /dev/null @@ -1,234 +0,0 @@ -"""BEVFusion PyTorch Pipeline Implementation with per-block latency.""" - -from __future__ import annotations - -import logging -import time -from typing import Dict, List, Tuple - -import torch -import torch.nn.functional as F -from typing_extensions import override - -from deployment.config.enums import Backend -from deployment.primitives.device import DeviceSpec -from deployment.projects.bevfusion.inference.bevfusion_inference_pipeline import BEVFusionDeploymentPipeline - -try: - from projects.BEVFusion.bevfusion.bevfusion import _ensure_float_for_pts_pipeline as _ensure_float_for_pts_impl -except Exception: - _ensure_float_for_pts_impl = None - -logger = logging.getLogger(__name__) - - -_PYTORCH_TENSOR_LOG_PREFIX = "[BEVFUSION][PyTorch][tensors]" - - -def _ensure_float_for_pts_pipeline(tensor: torch.Tensor) -> torch.Tensor: - """Best-effort compatibility wrapper for BEVFusion sparse feature dtype normalization.""" - if _ensure_float_for_pts_impl is not None: - return _ensure_float_for_pts_impl(tensor) - return tensor.float() if tensor.dtype != torch.float32 else tensor - - -def _tensor_stats(t: torch.Tensor, name: str) -> str: - """Return a compact string with tensor statistics for debugging.""" - t_f = t.float() - return ( - f"{_PYTORCH_TENSOR_LOG_PREFIX} {name}: shape={tuple(t.shape)} dtype={t.dtype} " - f"min={t_f.min().item():.4f} max={t_f.max().item():.4f} " - f"mean={t_f.mean().item():.4f} std={t_f.std().item():.4f} " - f"abs_mean={t_f.abs().mean().item():.4f} " - f"nonzero={t_f.count_nonzero().item()}/{t_f.numel()}" - ) - - -class BEVFusionPyTorchPipeline(BEVFusionDeploymentPipeline): - """PyTorch-based BEVFusion pipeline with per-block latency breakdown. - - Runs the full model natively, structured to match the ONNX/TensorRT - staged inference for output consistency. Reports latency for each block: - - voxel_encoder_ms: voxel mean reduction - - sparse_encoder_ms: pts_middle_encoder (spconv) - - backbone_ms: pts_backbone (SECOND) - - neck_ms: pts_neck (SECONDFPN) - - head_ms: bbox_head + postprocess scoring - """ - - _debug_frame_count = 0 - - def __init__(self, pytorch_model: torch.nn.Module, device: DeviceSpec) -> None: - super().__init__(pytorch_model=pytorch_model, backend_type=Backend.PYTORCH, device=device) - logger.info("BEVFusion PyTorch pipeline initialized (per-block latency enabled)") - - @override - def run_model( - self, - preprocessed_input: Dict[str, torch.Tensor], - ) -> Tuple[List[torch.Tensor], Dict[str, float]]: - """Run BEVFusion with per-block latency measurement. - - Breaks the model into stages and measures each one independently. - """ - stage_latencies: Dict[str, float] = {} - - total_start = time.perf_counter() - outputs = self._run_bevfusion_with_breakdown( - preprocessed_input["voxels"], - preprocessed_input["coors"], - preprocessed_input["num_points_per_voxel"], - stage_latencies, - ) - stage_latencies["bevfusion_ms"] = (time.perf_counter() - total_start) * 1000 - - return outputs, stage_latencies - - def _run_bevfusion_with_breakdown( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - stage_latencies: Dict[str, float], - ) -> List[torch.Tensor]: - """Run BEVFusion stage by stage, collecting per-block latencies.""" - model = self.pytorch_model - model.eval() - device = self.torch_device - - voxels = voxels.to(device) - coors = coors.to(device) - num_points_per_voxel = num_points_per_voxel.to(device) - - with torch.no_grad(): - # --- Stage 1: Voxel Encoder (mean reduction) --- - torch.cuda.synchronize() - t0 = time.perf_counter() - - if coors.shape[1] == 3: - num_points = coors.shape[0] - batch_coors = torch.zeros(num_points, 1, device=device, dtype=coors.dtype) - coors = torch.cat([batch_coors, coors], dim=1).contiguous() - - if getattr(model, "voxelize_reduce", True): - npt = num_points_per_voxel.type_as(voxels).view(-1, 1).clamp(min=1.0) - voxel_features = voxels.sum(dim=1, keepdim=False) / npt - else: - voxel_features = voxels - - torch.cuda.synchronize() - stage_latencies["voxel_encoder_ms"] = (time.perf_counter() - t0) * 1000 - - # --- Stage 2: Sparse Encoder (pts_middle_encoder / spconv) --- - torch.cuda.synchronize() - t1 = time.perf_counter() - - _dbg = BEVFusionPyTorchPipeline._debug_frame_count < 2 - BEVFusionPyTorchPipeline._debug_frame_count += 1 - if _dbg: - print( - f"{_PYTORCH_TENSOR_LOG_PREFIX} frame={BEVFusionPyTorchPipeline._debug_frame_count}/2 " - f"(native pts_middle_encoder → backbone → neck → head)" - ) - print(_tensor_stats(voxel_features, "voxel_features_input")) - - spatial_features = model.pts_middle_encoder(voxel_features, coors, batch_size=1) - spatial_features = _ensure_float_for_pts_pipeline(spatial_features) - - if _dbg: - print(_tensor_stats(spatial_features, "sparse_encoder_output")) - - torch.cuda.synchronize() - stage_latencies["sparse_encoder_ms"] = (time.perf_counter() - t1) * 1000 - - # --- Stage 3: Backbone (pts_backbone / SECOND) --- - torch.cuda.synchronize() - t2 = time.perf_counter() - - backbone_out = spatial_features - if hasattr(model, "pts_backbone") and model.pts_backbone is not None: - backbone_out = model.pts_backbone(_ensure_float_for_pts_pipeline(spatial_features)) - - if _dbg: - if isinstance(backbone_out, (list, tuple)): - for bi, bo in enumerate(backbone_out): - print(_tensor_stats(bo, f"backbone_out[{bi}]")) - else: - print(_tensor_stats(backbone_out, "backbone_out")) - - torch.cuda.synchronize() - stage_latencies["backbone_ms"] = (time.perf_counter() - t2) * 1000 - - # --- Stage 4: Neck (pts_neck / SECONDFPN) --- - torch.cuda.synchronize() - t3 = time.perf_counter() - - neck_out = backbone_out - if hasattr(model, "pts_neck") and model.pts_neck is not None: - neck_out = model.pts_neck(backbone_out) - - # Match ``BEVFusion.extract_feat``: head ``bev_pos`` is built for - # ``grid_size // out_size_factor`` (e.g. 180×180) while SECOND/FPN can - # yield full voxel BEV (e.g. 1440×1440). Skipping this pools causes - # ``key`` vs ``key_pos`` length mismatch in the transformer decoder. - align_fn = getattr(model, "_align_lidar_bev_to_head_grid", None) - if callable(align_fn): - neck_out = align_fn(neck_out) - - if _dbg: - if isinstance(neck_out, (list, tuple)): - for ni, no in enumerate(neck_out): - print(_tensor_stats(no, f"neck_out[{ni}]")) - else: - print(_tensor_stats(neck_out, "neck_out")) - - torch.cuda.synchronize() - stage_latencies["neck_ms"] = (time.perf_counter() - t3) * 1000 - - # --- Stage 5: Detection Head (bbox_head) --- - torch.cuda.synchronize() - t4 = time.perf_counter() - - preds = model.bbox_head(neck_out, []) - - torch.cuda.synchronize() - stage_latencies["head_ms"] = (time.perf_counter() - t4) * 1000 - - # --- Stage 6: Post-scoring --- - torch.cuda.synchronize() - t5 = time.perf_counter() - - preds = preds[0][0] - - if _dbg: - print(_tensor_stats(preds["heatmap"], "head_heatmap_raw")) - print(_tensor_stats(preds["center"][0], "head_center")) - print(_tensor_stats(preds["dim"][0], "head_dim")) - - score = preds["heatmap"].sigmoid() - one_hot = F.one_hot(preds["query_labels"], num_classes=score.size(1)).permute(0, 2, 1) - score = score * preds["query_heatmap_score"] * one_hot - score = score[0].max(dim=0)[0] - - bbox_pred = torch.cat( - [preds["center"][0], preds["height"][0], preds["dim"][0], preds["rot"][0], preds["vel"][0]], - dim=0, - ) - label_pred = preds["query_labels"][0] - - torch.cuda.synchronize() - stage_latencies["post_scoring_ms"] = (time.perf_counter() - t5) * 1000 - - return [bbox_pred, score, label_pred] - - @override - def run_bevfusion( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - ) -> List[torch.Tensor]: - # Not used: this pipeline overrides run_model() to call _run_bevfusion_with_breakdown() - # directly, so the base template never dispatches through run_bevfusion(). Present only to - # satisfy the abstract method on BEVFusionDeploymentPipeline. - raise NotImplementedError("BEVFusionPyTorchPipeline overrides run_model(); run_bevfusion() is not used.") diff --git a/deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py b/deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py deleted file mode 100644 index 9f85b1481..000000000 --- a/deployment/projects/bevfusion/inference/tensorrt_inference_pipeline.py +++ /dev/null @@ -1,808 +0,0 @@ -"""BEVFusion TensorRT Pipeline Implementation.""" - -from __future__ import annotations - -import logging -import os -import os.path as osp -from typing import Dict, List, Optional, Sequence, Tuple - -import numpy as np -import pycuda.autoinit # noqa: F401 -import pycuda.driver as cuda -import tensorrt as trt -import torch -from typing_extensions import override - -from deployment.config.enums import Backend -from deployment.config.schema import ComponentsConfig -from deployment.inference.gpu_resource_mixin import ( - GPUResourceMixin, - TensorRTResourceManager, - release_tensorrt_resources, -) -from deployment.primitives.artifacts import resolve_artifact_path -from deployment.primitives.device import DeviceSpec -from deployment.primitives.tensorrt_plugins import load_tensorrt_plugin_libraries -from deployment.projects.bevfusion.inference.bevfusion_inference_pipeline import BEVFusionDeploymentPipeline -from deployment.projects.bevfusion.inference.trt_profiling import ( - _SPARSE_BUCKET_ORDER, - _scale_dense_substages, - _sum_layers_by_stage, - _summarize_sparse_layers, - _TRTLayerProfiler, -) -from deployment.projects.bevfusion.io.component_utils import has_component, is_split_bevfusion_components -from deployment.projects.bevfusion.io.coors_contract import voxel_indices_xyz_to_graph_input_zyx - -logger = logging.getLogger(__name__) - - -def _env_truthy(key: str) -> bool: - return os.environ.get(key, "").strip().lower() in ("1", "true", "yes") - - -def _env_int(key: str, default: int) -> int: - try: - return int(os.environ.get(key, str(default)).strip()) - except ValueError: - return default - - -_TRT_DEBUG_SPLIT = _env_truthy("BEVFUSION_TRT_DEBUG_SPLIT") -_TRT_LOG_IO = _env_truthy("BEVFUSION_TRT_LOG_IO") -# Priority A — in-situ per-layer breakdown for the split sparse engine. -# BEVFUSION_TRT_SPARSE_PROFILE=1 attaches trt.IProfiler to the sparse context -# BEVFUSION_TRT_SPARSE_PROFILE_EVERY=1 log breakdown on every frame (default: every 10 frames) -# This env var is a sanity overlay when running the real eval path (step 5). -_TRT_SPARSE_PROFILE = _env_truthy("BEVFUSION_TRT_SPARSE_PROFILE") -_TRT_SPARSE_PROFILE_EVERY = max(1, _env_int("BEVFUSION_TRT_SPARSE_PROFILE_EVERY", 10)) -# First N eval frames: print pooled-voxel + lidar_bev stats to stdout (align with PyTorch pipeline). -_TRT_TENSOR_LOG_FRAMES = max(0, _env_int("BEVFUSION_TRT_TENSOR_LOG_FRAMES", 2)) -_TRT_TENSOR_LOG_PREFIX = "[BEVFUSION][TensorRT][tensors]" - - -def _np_tensor_stats(arr: np.ndarray, name: str) -> str: - """Compact numpy stats for debug lines (matches PyTorch _tensor_stats fields).""" - a = np.asarray(arr, dtype=np.float64).ravel() - nz = int(np.count_nonzero(a)) - return ( - f"{_TRT_TENSOR_LOG_PREFIX} {name}: shape={arr.shape} dtype={arr.dtype} " - f"min={float(a.min()):.4f} max={float(a.max()):.4f} " - f"mean={float(a.mean()):.4f} std={float(a.std()):.4f} " - f"abs_mean={float(np.mean(np.abs(a))):.4f} " - f"nonzero={nz}/{a.size}" - ) - - -def _list_trt_io_names(engine: trt.ICudaEngine) -> Tuple[List[str], List[str]]: - """Return (input_names, output_names) in TensorRT tensor index order.""" - inputs: List[str] = [] - outputs: List[str] = [] - for i in range(engine.num_io_tensors): - name = engine.get_tensor_name(i) - if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - inputs.append(name) - else: - outputs.append(name) - return inputs, outputs - - -def _pick_bound_input_name(engine: trt.ICudaEngine, expected_in_order: Sequence[str]) -> str: - """Match deploy_cfg input names to the engine; avoid relying on arbitrary TRT ordering.""" - found, _out = _list_trt_io_names(engine) - for want in expected_in_order: - if want in found: - return want - if len(found) == 1: - if expected_in_order and found[0] != expected_in_order[0]: - logger.warning( - "TensorRT dense engine input is %r but deploy_cfg expects %r — using engine binding. " - "If mAP=0, verify ONNX export names match deploy components.bevfusion_dense.io.inputs.", - found[0], - expected_in_order[0], - ) - return found[0] - raise RuntimeError(f"Could not map deploy_cfg inputs {list(expected_in_order)} to engine inputs {found}") - - -def _log_engine_schema(tag: str, engine: trt.ICudaEngine) -> None: - ins, outs = _list_trt_io_names(engine) - lines = [f"[trt-io] {tag} inputs={ins} outputs={outs}"] - for name in ins + outs: - shp = engine.get_tensor_shape(name) - dt = engine.get_tensor_dtype(name) - lines.append(f"[trt-io] {name}: shape={shp} dtype={dt}") - logger.warning("\n".join(lines)) - - -def _log_engine_input_dtypes_line(tag: str, engine: trt.ICudaEngine) -> None: - """Log binding dtypes (P1: FP32 vs FP16 voxels for split sparse engines). - - HALF voxel bindings are supported via ``_host_buffer_for_engine_tensor``; this line makes - the contract visible without enabling ``BEVFUSION_TRT_LOG_IO=1``. - """ - parts: List[str] = [] - for i in range(engine.num_io_tensors): - name = engine.get_tensor_name(i) - if engine.get_tensor_mode(name) != trt.TensorIOMode.INPUT: - continue - dt = engine.get_tensor_dtype(name) - parts.append(f"{name}={dt}") - ln = name.lower() - if "voxel" in ln and "num" not in ln and dt == trt.DataType.HALF: - logger.warning( - "[trt-io] %s: voxel-like input %r is HALF — host numpy is cast before " - "``set_tensor_address`` (see INFO ``casting host buffer`` on first infer). " - "If that cast is missing, ImplicitGemm inputs corrupt (lidar_bev explodes).", - tag, - name, - ) - if parts: - logger.info("[trt-io] %s engine INPUT dtypes: %s", tag, ", ".join(parts)) - - -class BEVFusionTensorRTPipeline(GPUResourceMixin, BEVFusionDeploymentPipeline): - """TensorRT-based BEVFusion pipeline. - - Single engine (full graph) or split sparse + dense engines. - """ - - def __init__( - self, - pytorch_model: torch.nn.Module, - tensorrt_dir: str, - device: DeviceSpec, - components_cfg: ComponentsConfig, - plugin_libraries: Tuple[str, ...] = (), - ) -> None: - super().__init__(pytorch_model=pytorch_model, backend_type=Backend.TENSORRT, device=device) - - self.tensorrt_dir = tensorrt_dir - self._components_cfg = components_cfg - self._plugin_libraries = plugin_libraries - self._trt_logger = trt.Logger(trt.Logger.WARNING) - split_layout = is_split_bevfusion_components(components_cfg) - merged_engine_available = False - if split_layout and has_component(components_cfg, "bevfusion_main_body"): - merged_engine_path = resolve_artifact_path( - base_dir=tensorrt_dir, - components_cfg=components_cfg, - component_name="bevfusion_main_body", - file_key="engine_file", - ) - merged_engine_available = osp.exists(merged_engine_path) - self._split = split_layout and not merged_engine_available - self._engine = None - self._context = None - self._engine_sparse = None - self._context_sparse = None - self._engine_dense = None - self._context_dense = None - - self._start_event = cuda.Event() - self._end_event = cuda.Event() - # Split-engine GPU intervals (same stream as each TRT execute, excludes D2H). - self._sparse_ev_s = cuda.Event() - self._sparse_ev_e = cuda.Event() - self._dense_ev_s = cuda.Event() - self._dense_ev_e = cuda.Event() - self._last_split_sparse_gpu_ms: float = 0.0 - self._last_split_dense_gpu_ms: float = 0.0 - self._split_debug_frames_done: int = 0 - self._split_debug_max: int = max(0, _env_int("BEVFUSION_TRT_DEBUG_SPLIT_FRAMES", 2)) - self._split_tensor_log_frames_done: int = 0 - # Priority A: accumulators for sparse encoder bucket breakdown across eval frames. - self._sparse_profile_frame_count: int = 0 - self._sparse_profile_bucket_sum: Dict[str, float] = {b: 0.0 for b in _SPARSE_BUCKET_ORDER} - self._sparse_profile_top_layers: Dict[str, float] = {} # name -> accumulated ms - self._last_sparse_profile_buckets: Dict[str, float] = {} - - self._load_tensorrt_engine() - logger.info(f"BEVFusion TensorRT pipeline initialized from: {tensorrt_dir} (split={self._split})") - - def _load_tensorrt_engine(self) -> None: - load_tensorrt_plugin_libraries(logger, self._plugin_libraries) - trt.init_libnvinfer_plugins(self._trt_logger, "") - runtime = trt.Runtime(self._trt_logger) - - if self._split: - sparse_path = resolve_artifact_path( - base_dir=self.tensorrt_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_sparse", - file_key="engine_file", - ) - dense_path = resolve_artifact_path( - base_dir=self.tensorrt_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_dense", - file_key="engine_file", - ) - if not osp.exists(sparse_path): - raise FileNotFoundError(f"Sparse TensorRT engine not found: {sparse_path}") - if not osp.exists(dense_path): - raise FileNotFoundError(f"Dense TensorRT engine not found: {dense_path}") - - with open(sparse_path, "rb") as f: - self._engine_sparse = runtime.deserialize_cuda_engine(f.read()) - with open(dense_path, "rb") as f: - self._engine_dense = runtime.deserialize_cuda_engine(f.read()) - if self._engine_sparse is None or self._engine_dense is None: - raise RuntimeError("Failed to deserialize split TensorRT engines") - - self._context_sparse = self._engine_sparse.create_execution_context() - self._context_dense = self._engine_dense.create_execution_context() - if self._context_sparse is None or self._context_dense is None: - raise RuntimeError("Failed to create TensorRT contexts for split engines") - logger.info("Loaded split TensorRT engines: %s , %s", sparse_path, dense_path) - assert self._engine_sparse is not None and self._engine_dense is not None - _log_engine_input_dtypes_line("bevfusion_sparse", self._engine_sparse) - _log_engine_input_dtypes_line("bevfusion_dense", self._engine_dense) - if _TRT_LOG_IO: - _log_engine_schema("bevfusion_sparse", self._engine_sparse) - _log_engine_schema("bevfusion_dense", self._engine_dense) - return - - engine_path = resolve_artifact_path( - base_dir=self.tensorrt_dir, - components_cfg=self._components_cfg, - component_name="bevfusion_main_body", - file_key="engine_file", - ) - if not osp.exists(engine_path): - raise FileNotFoundError(f"TensorRT engine not found: {engine_path}") - - with open(engine_path, "rb") as f: - self._engine = runtime.deserialize_cuda_engine(f.read()) - if self._engine is None: - raise RuntimeError(f"Failed to deserialize engine: {engine_path}") - - self._context = self._engine.create_execution_context() - if self._context is None: - raise RuntimeError("Failed to create TensorRT execution context (OOM?)") - - logger.info(f"Loaded TensorRT engine: {engine_path}") - - @staticmethod - def _trt_dtype_to_numpy(trt_dtype: trt.DataType) -> np.dtype: - """Map TensorRT dtype to numpy dtype for correctly sized host buffers.""" - try: - return np.dtype(trt.nptype(trt_dtype)) - except Exception: - # Safe fallback for older/newer TRT dtype variations. - mapping = {} - for key, npdt in ( - ("FLOAT", np.float32), - ("HALF", np.float16), - ("INT8", np.int8), - ("INT32", np.int32), - ("BOOL", np.bool_), - ("UINT8", np.uint8), - ("FP8", np.float16), - ("BF16", np.float16), - ("INT64", np.int64), - ): - dt = getattr(trt.DataType, key, None) - if dt is not None: - mapping[dt] = npdt - return np.dtype(mapping.get(trt_dtype, np.float32)) - - def _host_buffer_for_engine_tensor(self, engine: trt.ICudaEngine, tensor_name: str, arr: np.ndarray) -> np.ndarray: - """Cast / layout host memory to match *engine* binding dtype (critical for FP16 engines). - - Split sparse ONNX is often traced with FP32 voxels, but TensorRT ``fp16`` builds may bind - ``voxels`` as ``HALF``. Feeding float32 nbytes into a HALF binding misaligns the GPU - buffer and corrupts the first ImplicitGemm inputs (lidar_bev explosion while numpy - voxel stats still look sane). - """ - trt_dtype = engine.get_tensor_dtype(tensor_name) - want = self._trt_dtype_to_numpy(trt_dtype) - if arr.dtype != want: - logger.info( - "[trt-io] casting host buffer for tensor %r: numpy %s → %s (engine binding %s)", - tensor_name, - arr.dtype, - want, - trt_dtype, - ) - arr = np.asarray(arr, dtype=want) - if not arr.flags["C_CONTIGUOUS"]: - arr = np.ascontiguousarray(arr) - return arr - - def _trt_infer_voxel_inputs( - self, - engine: trt.ICudaEngine, - context: trt.IExecutionContext, - voxels_np: np.ndarray, - coors_np: np.ndarray, - num_points_np: np.ndarray, - profiler: Optional[_TRTLayerProfiler], - gpu_interval_events: Optional[Tuple[cuda.Event, cuda.Event]], - ) -> Dict[str, np.ndarray]: - input_map: Dict[str, np.ndarray] = {} - output_names: List[str] = [] - for i in range(engine.num_io_tensors): - name = engine.get_tensor_name(i) - if engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT: - ln = name.lower() - if "voxel" in ln and "num" not in ln: - input_map[name] = voxels_np - elif "coor" in ln: - input_map[name] = coors_np - elif "num" in ln: - input_map[name] = num_points_np - else: - output_names.append(name) - return self._trt_infer_bound(engine, context, input_map, output_names, profiler, gpu_interval_events) - - def _trt_infer_named_input( - self, - engine: trt.ICudaEngine, - context: trt.IExecutionContext, - input_map: Dict[str, np.ndarray], - profiler: Optional[_TRTLayerProfiler], - gpu_interval_events: Optional[Tuple[cuda.Event, cuda.Event]], - ) -> Dict[str, np.ndarray]: - output_names: List[str] = [] - for i in range(engine.num_io_tensors): - name = engine.get_tensor_name(i) - if engine.get_tensor_mode(name) == trt.TensorIOMode.OUTPUT: - output_names.append(name) - return self._trt_infer_bound(engine, context, input_map, output_names, profiler, gpu_interval_events) - - def _trt_infer_bound( - self, - engine: trt.ICudaEngine, - context: trt.IExecutionContext, - input_map: Dict[str, np.ndarray], - output_names: List[str], - profiler: Optional[_TRTLayerProfiler], - gpu_interval_events: Optional[Tuple[cuda.Event, cuda.Event]], - ) -> Dict[str, np.ndarray]: - input_map = {name: self._host_buffer_for_engine_tensor(engine, name, arr) for name, arr in input_map.items()} - for name, arr in input_map.items(): - context.set_input_shape(name, arr.shape) - - output_arrays: Dict[str, np.ndarray] = {} - for name in output_names: - shape = context.get_tensor_shape(name) - trt_dtype = engine.get_tensor_dtype(name) - np_dtype = self._trt_dtype_to_numpy(trt_dtype) - arr = np.empty(shape, dtype=np_dtype) - if not arr.flags["C_CONTIGUOUS"]: - arr = np.ascontiguousarray(arr) - output_arrays[name] = arr - - prev_profiler = None - if profiler is not None and hasattr(context, "profiler"): - prev_profiler = getattr(context, "profiler", None) - context.profiler = profiler - profiler.layer_times.clear() - - try: - with TensorRTResourceManager() as mgr: - d_inputs = {name: mgr.allocate(arr.nbytes) for name, arr in input_map.items()} - d_outputs = {name: mgr.allocate(arr.nbytes) for name, arr in output_arrays.items()} - stream = mgr.stream - - for name, arr in input_map.items(): - context.set_tensor_address(name, int(d_inputs[name])) - cuda.memcpy_htod_async(d_inputs[name], arr, stream) - - for name in output_names: - context.set_tensor_address(name, int(d_outputs[name])) - - if gpu_interval_events is not None: - gpu_interval_events[0].record(stream) - ok = context.execute_async_v3(stream_handle=stream.handle) - if not ok: - raise RuntimeError("TensorRT execute_async_v3 returned failure status.") - if gpu_interval_events is not None: - gpu_interval_events[1].record(stream) - - for name in output_names: - cuda.memcpy_dtoh_async(output_arrays[name], d_outputs[name], stream) - - mgr.synchronize() - finally: - # TensorRT rejects setProfiler(nullptr); only restore when a previous profiler existed. - if profiler is not None and hasattr(context, "profiler") and prev_profiler is not None: - context.profiler = prev_profiler - - return output_arrays - - @override - def run_bevfusion( - self, - voxels: torch.Tensor, - coors: torch.Tensor, - num_points_per_voxel: torch.Tensor, - profiler: _TRTLayerProfiler | None = None, - ) -> List[torch.Tensor]: - voxels_np = self.to_numpy(voxels, dtype=np.float32) - coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) - num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) - # Match ``extract_pts_feat``: mean-pool must not divide by zero (NaN BEV → dense NaN). - num_points_np = np.maximum(num_points_np, 1) - - if self._split: - assert self._engine_sparse is not None and self._context_sparse is not None - assert self._engine_dense is not None and self._context_dense is not None - - sparse_cfg = self._components_cfg.get_component("bevfusion_sparse") - dense_cfg = self._components_cfg.get_component("bevfusion_dense") - exp_sparse_out = [o.name for o in sparse_cfg.io.outputs] - exp_dense_in = [i.name for i in dense_cfg.io.inputs] - - do_tensor_log = _TRT_TENSOR_LOG_FRAMES > 0 and self._split_tensor_log_frames_done < _TRT_TENSOR_LOG_FRAMES - if do_tensor_log: - self._split_tensor_log_frames_done += 1 - fi = self._split_tensor_log_frames_done - print( - f"{_TRT_TENSOR_LOG_PREFIX} frame={fi}/{_TRT_TENSOR_LOG_FRAMES} " - f"(sparse TRT engine → lidar_bev → dense TRT engine → bbox/score/label)" - ) - voxelize_reduce = getattr(self.pytorch_model, "voxelize_reduce", True) - if voxelize_reduce and voxels_np.ndim == 3: - # Match pytorch.py: [N,P,C].sum(1) / npt with npt [N,1] → [N,C] (not [N] / [N,C]). - npt = np.maximum(num_points_np.astype(np.float32).reshape(-1, 1), 1.0) - voxel_feat_np = voxels_np.sum(axis=1, keepdims=False) / npt - print(_np_tensor_stats(voxel_feat_np, "voxel_features_input (numpy mean-pool, same as PyTorch)")) - elif voxels_np.ndim == 2: - print( - _np_tensor_stats( - voxels_np, - "voxel_features_input (already [N,C], no per-point dim — same as fed to TRT)", - ) - ) - else: - print( - f"{_TRT_TENSOR_LOG_PREFIX} voxel_features_input: skipped " - f"(voxelize_reduce={voxelize_reduce}, voxels_ndim={voxels_np.ndim})" - ) - - # Sparse (spconv) engine: CUDA-timed separately. If BEVFUSION_TRT_SPARSE_PROFILE=1, - # also attach a dedicated IProfiler here so we can answer Priority A's question - # ("where does the sparse time actually go?") without a separate run. - sparse_profiler: Optional[_TRTLayerProfiler] = _TRTLayerProfiler() if _TRT_SPARSE_PROFILE else None - sparse_out = self._trt_infer_voxel_inputs( - self._engine_sparse, - self._context_sparse, - voxels_np, - coors_np, - num_points_np, - profiler=sparse_profiler, - gpu_interval_events=(self._sparse_ev_s, self._sparse_ev_e), - ) - if sparse_profiler is not None: - self._record_sparse_profile(sparse_profiler.layer_times) - if len(sparse_out) != 1: - raise RuntimeError(f"Sparse engine: expected 1 output, got {list(sparse_out.keys())}") - bev_name = next(iter(sparse_out)) - if exp_sparse_out and bev_name not in exp_sparse_out: - logger.warning( - "[trt-split] sparse engine output tensor is %r but deploy_cfg bevfusion_sparse.io.outputs " - "names=%s — check ONNX export / TRT binding names.", - bev_name, - exp_sparse_out, - ) - bev_arr = np.ascontiguousarray(sparse_out[bev_name].astype(np.float32)) - - if do_tensor_log: - bn = bev_arr.reshape(-1) - print(_np_tensor_stats(bev_arr, f"sparse_encoder_output ({bev_name}, TRT sparse engine)")) - if bool(np.isnan(bn).any()) or bool(np.isinf(bn).any()): - print( - f"{_TRT_TENSOR_LOG_PREFIX} WARNING: lidar_bev has nan={bool(np.isnan(bn).any())} " - f"inf={bool(np.isinf(bn).any())}" - ) - - do_split_dbg = _TRT_DEBUG_SPLIT and self._split_debug_frames_done < self._split_debug_max - if do_split_dbg: - self._split_debug_frames_done += 1 - if not do_tensor_log: - bn = bev_arr.reshape(-1) - logger.warning( - "[BEVFUSION][TensorRT][debug-split] frame=%d/%d sparse->dense %s: shape=%s dtype=%s " - "min=%.6f max=%.6f mean=%.6f std=%.6f abs_mean=%.6f nan=%s inf=%s", - self._split_debug_frames_done, - self._split_debug_max, - bev_name, - bev_arr.shape, - bev_arr.dtype, - float(bn.min()), - float(bn.max()), - float(bn.mean()), - float(bn.std()), - float(np.mean(np.abs(bn))), - bool(np.isnan(bn).any()), - bool(np.isinf(bn).any()), - ) - - dense_in_name = _pick_bound_input_name(self._engine_dense, exp_dense_in) - - if do_split_dbg: - ctx = self._context_dense - exp_shape = tuple(ctx.get_tensor_shape(dense_in_name)) - logger.warning( - "[BEVFUSION][TensorRT][debug-split] dense input %r engine_expected_shape=%s feed_shape=%s " - "deploy_cfg_inputs=%s", - dense_in_name, - exp_shape, - bev_arr.shape, - exp_dense_in, - ) - if tuple(bev_arr.shape) != exp_shape and not any(d < 0 for d in exp_shape): - logger.warning( - "[BEVFUSION][TensorRT][debug-split] SHAPE MISMATCH: lidar_bev numpy shape %s vs TRT context %s — " - "dense engine will error or broadcast wrong; common cause: H×W vs export grid.", - bev_arr.shape, - exp_shape, - ) - - dense_out = self._trt_infer_named_input( - self._engine_dense, - self._context_dense, - {dense_in_name: bev_arr}, - profiler, - gpu_interval_events=(self._dense_ev_s, self._dense_ev_e), - ) - - self._sparse_ev_e.synchronize() - self._dense_ev_e.synchronize() - self._last_split_sparse_gpu_ms = float(self._sparse_ev_e.time_since(self._sparse_ev_s)) - self._last_split_dense_gpu_ms = float(self._dense_ev_e.time_since(self._dense_ev_s)) - - expected_output_names = [out.name for out in dense_cfg.io.outputs] - out_keys = list(dense_out.keys()) - ordered_names = [n for n in expected_output_names if n in dense_out] - ordered_names += [n for n in out_keys if n not in ordered_names] - tensors = [torch.from_numpy(dense_out[name]).to(self.torch_device) for name in ordered_names] - if do_tensor_log and tensors: - for i, name in enumerate(ordered_names): - t = tensors[i].detach() - t_f = t.float().reshape(-1) - extra = "" - if name == "bbox_pred" and t.ndim >= 2 and t.shape[0] >= 2: - cx = t[0].float().reshape(-1) - cy = t[1].float().reshape(-1) - extra = ( - f" center_x[min,max]=({float(cx.min()):.4f},{float(cx.max()):.4f}) " - f"center_y[min,max]=({float(cy.min()):.4f},{float(cy.max()):.4f})" - ) - if name == "label_pred": - lp = t.reshape(-1).long() - uniq = torch.unique(lp) - extra = ( - f" label_unique_count={int(uniq.numel())} label_min={int(lp.min())} " - f"label_max={int(lp.max())}" - ) - if name == "score": - extra = ( - f" score>0.1_count={int((t_f > 0.1).sum())} " f"score>0.5_count={int((t_f > 0.5).sum())}" - ) - print( - f"{_TRT_TENSOR_LOG_PREFIX} dense_out[{i}] {name} (TRT dense engine): " - f"shape={tuple(t.shape)} dtype={t.dtype} " - f"min={float(t_f.min().item()):.4f} max={float(t_f.max().item()):.4f} " - f"mean={float(t_f.mean().item()):.4f}{extra}" - ) - elif do_split_dbg and tensors: - for i, name in enumerate(ordered_names): - t = tensors[i].detach() - t_f = t.float().reshape(-1) - extra = "" - if name == "bbox_pred" and t.ndim >= 2 and t.shape[0] >= 2: - cx = t[0].float().reshape(-1) - cy = t[1].float().reshape(-1) - extra = f" center_x[min,max]=({float(cx.min())},{float(cx.max())}) center_y[min,max]=({float(cy.min())},{float(cy.max())})" - if name == "label_pred": - lp = t.reshape(-1).long() - uniq = torch.unique(lp) - extra = f" label_unique_count={int(uniq.numel())} label_min={int(lp.min())} label_max={int(lp.max())}" - if name == "score": - extra = f" score>0.1_count={int((t_f > 0.1).sum())} score>0.5_count={int((t_f > 0.5).sum())}" - logger.warning( - "[BEVFUSION][TensorRT][debug-split] dense_out[%s] %s: shape=%s dtype=%s min=%.6f max=%.6f mean=%.6f%s", - i, - name, - tuple(t.shape), - t.dtype, - float(t_f.min().item()), - float(t_f.max().item()), - float(t_f.mean().item()), - extra, - ) - return tensors - - engine = self._engine - context = self._context - assert engine is not None and context is not None - - output_arrays = self._trt_infer_voxel_inputs( - engine, - context, - voxels_np, - coors_np, - num_points_np, - profiler, - gpu_interval_events=(self._start_event, self._end_event), - ) - output_names = list(output_arrays.keys()) - - component_cfg = self._components_cfg.get_component("bevfusion_main_body") - expected_output_names = [out.name for out in component_cfg.io.outputs] - ordered_names = [n for n in expected_output_names if n in output_arrays] - ordered_names += [n for n in output_names if n not in ordered_names] - return [torch.from_numpy(output_arrays[name]).to(self.torch_device) for name in ordered_names] - - # Stage keys aligned with BEVFusionPyTorchPipeline for consistent Stage-wise Latency Breakdown. - # ``dense_engine_ms`` is the dense branch GPU time (split: CUDA events, merged: derived residual). - BEVFUSION_STAGE_KEYS = ( - "voxel_encoder_ms", - "sparse_encoder_ms", - "dense_engine_ms", - "backbone_ms", - "neck_ms", - "head_ms", - "post_scoring_ms", - "dense_unattributed_ms", - "bevfusion_ms", - ) - - @override - def run_model(self, preprocessed_input: Dict[str, torch.Tensor]) -> Tuple[List[torch.Tensor], Dict[str, float]]: - stage_latencies: Dict[str, float] = {k: 0.0 for k in self.BEVFUSION_STAGE_KEYS} - - profiler = _TRTLayerProfiler() - outputs = self.run_bevfusion( - preprocessed_input["voxels"], - preprocessed_input["coors"], - preprocessed_input["num_points_per_voxel"], - profiler=profiler, - ) - - # ------------------------------------------------------------------ - # Step 1: authoritative top-line GPU intervals (CUDA events). - # - bevfusion_ms : total TRT GPU time for the BEVFusion model. - # - sparse_encoder_ms / dense_engine_ms : the two top-level branches. - # Split has two physical engines (separate CUDA-event intervals). Merged - # is one engine, so we split its single interval by the per-layer profiler - # proportions (same classifier as the sub-stages below) — keeping every - # number on one clock and one naming contract. - # ------------------------------------------------------------------ - stage_sums = _sum_layers_by_stage(profiler.layer_times) if profiler.layer_times else None - - if self._split: - sparse_ms = self._last_split_sparse_gpu_ms - dense_ms = self._last_split_dense_gpu_ms - stage_latencies["bevfusion_ms"] = sparse_ms + dense_ms - else: - self._end_event.synchronize() - bevfusion_ms = float(self._end_event.time_since(self._start_event)) - stage_latencies["bevfusion_ms"] = bevfusion_ms - if stage_sums is not None: - total_raw = sum(stage_sums.values()) - sparse_frac = (stage_sums["sparse_encoder_ms"] / total_raw) if total_raw > 0.0 else 0.0 - sparse_ms = bevfusion_ms * sparse_frac - else: - sparse_ms = 0.0 - dense_ms = max(bevfusion_ms - sparse_ms, 0.0) - - stage_latencies["sparse_encoder_ms"] = sparse_ms - stage_latencies["dense_engine_ms"] = dense_ms - - # ------------------------------------------------------------------ - # Step 2: dense sub-stage breakdown — IDENTICAL path for merged & split. - # Per-layer (order-independent) classification gives the relative weight - # of backbone/neck/head/post_scoring, rescaled to the dense GPU interval. - # ------------------------------------------------------------------ - if stage_sums is not None: - dense_dist = _scale_dense_substages(stage_sums, dense_ms) - stage_latencies["backbone_ms"] = dense_dist["backbone_ms"] - stage_latencies["neck_ms"] = dense_dist["neck_ms"] - stage_latencies["head_ms"] = dense_dist["head_ms"] - stage_latencies["post_scoring_ms"] = dense_dist["post_scoring_ms"] - stage_latencies["dense_unattributed_ms"] = dense_dist["dense_unattributed_ms"] - else: - stage_latencies["dense_unattributed_ms"] = dense_ms - - # Align "Model" with the same interval semantics across merged/split TensorRT: - # report model_ms as the BEVFusion TRT GPU segment (not wall-clock Python overhead). - stage_latencies["model_ms"] = stage_latencies.get("bevfusion_ms", 0.0) - - return outputs, stage_latencies - - def _record_sparse_profile(self, layer_times: List[Tuple[str, float]]) -> None: - """Priority A in-situ overlay: accumulate sparse-engine bucket breakdown. - - We keep running sums across all eval frames so that after the run the user can - read off a 'mean sparse encoder bucket' right next to the normal latency table. - """ - if not layer_times: - return - buckets = _summarize_sparse_layers(layer_times) - self._last_sparse_profile_buckets = buckets - self._sparse_profile_frame_count += 1 - for b, ms in buckets.items(): - self._sparse_profile_bucket_sum[b] = self._sparse_profile_bucket_sum.get(b, 0.0) + ms - for name, ms in layer_times: - self._sparse_profile_top_layers[name] = self._sparse_profile_top_layers.get(name, 0.0) + ms - - if self._sparse_profile_frame_count % _TRT_SPARSE_PROFILE_EVERY == 0: - total = sum(buckets.values()) or 1e-9 - parts = [ - f"{b}={buckets[b]:.3f}ms ({buckets[b] / total * 100.0:.1f}%)" - for b in _SPARSE_BUCKET_ORDER - if buckets.get(b, 0.0) > 0.0 - ] - logger.info( - "[priority-a][sparse-profile] frame=%d sparse_layer_sum=%.3fms | %s", - self._sparse_profile_frame_count, - total, - " ".join(parts), - ) - - def print_sparse_profile_summary(self) -> None: - """Print Priority A mean-per-frame sparse-engine bucket breakdown. - - Called by the evaluator at the end of the run; no-op if the env var was off. - """ - n = self._sparse_profile_frame_count - if n <= 0: - return - logger.info("=" * 72) - logger.info("[priority-a] Sparse encoder in-situ bucket breakdown (mean/frame, n=%d)", n) - logger.info("=" * 72) - total_mean = sum(self._sparse_profile_bucket_sum.values()) / n - for b in _SPARSE_BUCKET_ORDER: - s = self._sparse_profile_bucket_sum.get(b, 0.0) - if s <= 0.0: - continue - mean = s / n - pct = (s / (total_mean * n)) * 100.0 if total_mean > 0.0 else 0.0 - logger.info(" %-20s %8.3f ms (%5.2f%%)", b, mean, pct) - logger.info(" %-20s %8.3f ms", "SUM", total_mean) - top_items = sorted(self._sparse_profile_top_layers.items(), key=lambda kv: -kv[1])[:10] - logger.info("Top 10 sparse layers (mean/frame):") - for name, acc in top_items: - logger.info(" %8.3f ms %s", acc / n, name) - logger.info("=" * 72) - - def _release_gpu_resources(self) -> None: - # Priority A — emit the sparse-profile summary before we tear engines down. - try: - self.print_sparse_profile_summary() - except Exception as exc: - logger.warning("[priority-a] sparse-profile summary failed: %s", exc) - for attr in ( - "_start_event", - "_end_event", - "_sparse_ev_s", - "_sparse_ev_e", - "_dense_ev_s", - "_dense_ev_e", - ): - if hasattr(self, attr): - try: - delattr(self, attr) - except Exception: - pass - if self._split: - release_tensorrt_resources( - engines={ - "sparse": self._engine_sparse, - "dense": self._engine_dense, - }, - contexts={ - "sparse": self._context_sparse, - "dense": self._context_dense, - }, - ) - else: - release_tensorrt_resources( - engines={"main": self._engine} if self._engine else None, - contexts={"main": self._context} if self._context else None, - ) diff --git a/deployment/projects/bevfusion/inference/trt_profiling.py b/deployment/projects/bevfusion/inference/trt_profiling.py deleted file mode 100644 index 3a678f01d..000000000 --- a/deployment/projects/bevfusion/inference/trt_profiling.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -"""Per-layer TensorRT profiling + BEVFusion stage attribution. - -Pure, order-independent helpers used by :class:`~...tensorrt_inference_pipeline.BEVFusionTensorRTPipeline` -to turn a TRT layer-time list into (a) Priority-A sparse-encoder buckets and (b) BEVFusion stage sums -(sparse / backbone / neck / head / post-scoring). Kept out of the inference pipeline so the runtime -path is not interleaved with ~200 lines of profiling/attribution logic. -""" - -import re -from typing import Dict, List, Tuple - -import tensorrt as trt - - -class _TRTLayerProfiler(trt.IProfiler): - """Collects per-layer execution times for TensorRT engine.""" - - def __init__(self) -> None: - try: - trt.IProfiler.__init__(self) - except Exception: - pass - self.layer_times: List[Tuple[str, float]] = [] - - def report_layer_time(self, layer_name: str, ms: float) -> None: - self.layer_times.append((str(layer_name), float(ms))) - - -# Priority A — bucket classification for sparse encoder in-situ profile. -_SPARSE_BUCKET_ORDER: Tuple[str, ...] = ( - "pair_gen", - "implicit_gemm_fp", - "scatter_nd", - "add", - "relu", - "cast", - "layout", - "other", -) - - -def _classify_sparse_bucket(layer_name: str) -> str: - """Match sparse encoder TRT layer names to a Priority A bucket. - - Keep patterns simple & case-insensitive — TRT forwards ONNX node names with - occasional prefixes, so pure substring matching is enough and cheap. - """ - n = layer_name.lower() - # Normalize common separators so ``ImplicitGemm`` / ``implicit-gemm`` / ``implicit gemm`` - # all collapse to ``implicitgemm`` before substring matching. - n_norm = n.replace("_", "").replace("-", "").replace(" ", "") - if "getindicepairsimplicitgemm" in n_norm or ("getindicepairs" in n_norm and "implicitgemm" not in n_norm): - return "pair_gen" - if "implicitgemm" in n_norm or "indiceconv" in n_norm: - return "implicit_gemm_fp" - if "scatternd" in n: - return "scatter_nd" - # Guard: "add"/"relu"/"cast" must be word-like to avoid matching paths. - if "relu" in n: - return "relu" - if "/add" in n or n.endswith("_add") or n.startswith("add"): - return "add" - if "/cast" in n or "_cast_" in n or n.startswith("cast"): - return "cast" - if any(k in n for k in ("reshape", "transpose", "concat", "slice", "gather", "squeeze", "unsqueeze")): - return "layout" - return "other" - - -def _summarize_sparse_layers(layer_times: List[Tuple[str, float]]) -> Dict[str, float]: - """Sum sparse TRT layer times per Priority A bucket (ms per frame).""" - sums: Dict[str, float] = {b: 0.0 for b in _SPARSE_BUCKET_ORDER} - for layer_name, ms in layer_times: - sums[_classify_sparse_bucket(layer_name)] += ms - return sums - - -# ============================================================================ -# Unified BEVFusion stage attribution (merged & split use the SAME logic). -# ---------------------------------------------------------------------------- -# Grounded in the BEVFusion ONNX module hierarchy, which is IDENTICAL for the -# merged full graph and the split dense graph (the merged graph only adds -# ``sparse/`` and ``dense/`` prefixes): -# pts_middle_encoder / spconv / ImplicitGemm ... -> sparse encoder -# pts_backbone (``blocks``) -> backbone -# pts_neck (``deblocks``) -> neck -# bbox_head (decoder / prediction_heads / heatmap_head) -> head -# score ops (sigmoid / one_hot / query_*) -> post scoring -# -# Classification is per-layer and ORDER-INDEPENDENT. This is the critical -# property: TensorRT freely fuses/reorders layers, so the previous order-based -# state machine mis-attributed cost (e.g. one early ``bbox_head`` layer flipped -# the whole stream to "head" and starved backbone/neck). A pure substring -# bucket per layer is stable regardless of fusion/order. -# ============================================================================ - -_BEVFUSION_DENSE_SUBSTAGE_KEYS: Tuple[str, ...] = ( - "backbone_ms", - "neck_ms", - "head_ms", - "post_scoring_ms", -) -_STAGE_OTHER = "other_ms" - - -def _classify_bevfusion_layer(layer_name: str) -> str: - """Classify one TensorRT layer into a BEVFusion stage key (order-independent). - - Returns one of: ``sparse_encoder_ms``, ``backbone_ms``, ``neck_ms``, - ``head_ms``, ``post_scoring_ms``, or ``other_ms`` (shape/glue, ~0 GPU time). - """ - n = layer_name.lower() - # Normalize separators so ``ImplicitGemm``/``implicit_gemm``/``getindicepairs`` all match. - nn = n.replace("_", "").replace("-", "").replace(" ", "") - - if any( - k in n - for k in ( - "pts_middle_encoder", - "middle_encoder", - "spconv", - "sparse_conv", - "subm", - "encoder_layer", - "conv_input", - "conv_out", - ) - ) or any(k in nn for k in ("implicitgemm", "getindicepairs", "scatternd")): - return "sparse_encoder_ms" - - # Neck before backbone: ``deblocks`` contains the substring ``blocks``. - if "pts_neck" in n or "deblocks" in n: - return "neck_ms" - - if "pts_backbone" in n or re.search(r"(^|[/.])blocks([/.]|$)", n): - return "backbone_ms" - - # ``bbox_head`` covers the transformer decoder, prediction_heads and heatmap_head. - if "bbox_head" in n: - return "head_ms" - - # Post-scoring ops typically live OUTSIDE bbox_head (top-level sigmoid/one_hot/query_*). - if any(k in nn for k in ("queryheatmapscore", "querylabels", "onehot")) or any( - k in n for k in ("sigmoid", "/topk", "argmax") - ): - return "post_scoring_ms" - - return _STAGE_OTHER - - -def _sum_layers_by_stage(layer_times: List[Tuple[str, float]]) -> Dict[str, float]: - """Sum profiler layer times into BEVFusion stage buckets (order-independent).""" - sums: Dict[str, float] = { - "sparse_encoder_ms": 0.0, - "backbone_ms": 0.0, - "neck_ms": 0.0, - "head_ms": 0.0, - "post_scoring_ms": 0.0, - _STAGE_OTHER: 0.0, - } - for layer_name, ms in layer_times: - sums[_classify_bevfusion_layer(layer_name)] += ms - return sums - - -def _scale_dense_substages(stage_sums: Dict[str, float], dense_total_ms: float) -> Dict[str, float]: - """Distribute the (CUDA-timed) dense total across backbone/neck/head/post_scoring. - - The per-layer profiler sums give the RELATIVE weight of each dense stage; we - rescale them so they add up exactly to ``dense_total_ms`` (the authoritative - GPU interval). ``other`` (shape/glue, ~0 GPU time) is absorbed proportionally, - so ``dense_unattributed_ms`` stays 0 whenever named stages are present. - """ - out: Dict[str, float] = {k: 0.0 for k in _BEVFUSION_DENSE_SUBSTAGE_KEYS} - out["dense_unattributed_ms"] = 0.0 - if dense_total_ms <= 0.0: - return out - - named_sum = sum(stage_sums.get(k, 0.0) for k in _BEVFUSION_DENSE_SUBSTAGE_KEYS) - if named_sum > 0.0: - scale = dense_total_ms / named_sum - for k in _BEVFUSION_DENSE_SUBSTAGE_KEYS: - out[k] = stage_sums.get(k, 0.0) * scale - out["dense_unattributed_ms"] = 0.0 - else: - out["dense_unattributed_ms"] = dense_total_ms - return out diff --git a/deployment/projects/bevfusion/io/component_utils.py b/deployment/projects/bevfusion/io/component_utils.py deleted file mode 100644 index 1ee0f4467..000000000 --- a/deployment/projects/bevfusion/io/component_utils.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Helpers for BEVFusion deploy ``components`` layout.""" - -from __future__ import annotations - -from typing import Any, Mapping - -from deployment.config.schema import ComponentsConfig - - -def is_split_bevfusion_components(components_cfg: ComponentsConfig) -> bool: - """True when deploy config uses sparse + dense ONNX/TRT (route 1), not a single main_body.""" - names = set(components_cfg.component_names()) - return "bevfusion_sparse" in names and "bevfusion_dense" in names - - -def should_merge_split_bevfusion(deploy_cfg: Mapping[str, Any]) -> bool: - """Return True when deploy config requests split->single export/eval merge.""" - merge_raw = deploy_cfg.get("bevfusion_merge", deploy_cfg.get("merge_bevfusion", deploy_cfg.get("merge", False))) - if isinstance(merge_raw, Mapping): - return bool(merge_raw.get("enabled", False)) - return bool(merge_raw) - - -def has_component(components_cfg: ComponentsConfig, component_name: str) -> bool: - """Return True if the component exists.""" - try: - components_cfg.get_component(component_name) - return True - except KeyError: - return False - - -def maybe_add_merged_main_body_component( - *, - deploy_cfg: Mapping[str, Any], - components_cfg: ComponentsConfig, -) -> ComponentsConfig: - """Optionally add merged main_body component while keeping split components. - - When ``bevfusion_merge`` is enabled and components are split, this function adds - ``bevfusion_main_body`` by reusing: - - split sparse input schema / TensorRT profile - - split dense output schema - """ - if not is_split_bevfusion_components(components_cfg): - return components_cfg - if not should_merge_split_bevfusion(deploy_cfg): - return components_cfg - if has_component(components_cfg, "bevfusion_main_body"): - return components_cfg - - sparse_cfg = components_cfg.get_component("bevfusion_sparse") - dense_cfg = components_cfg.get_component("bevfusion_dense") - - merge_raw = deploy_cfg.get("bevfusion_merge", deploy_cfg.get("merge_bevfusion", deploy_cfg.get("merge", {}))) - merge_cfg = merge_raw if isinstance(merge_raw, Mapping) else {} - onnx_file = str(merge_cfg.get("onnx_file", "bevfusion_lidar.onnx")) - engine_file = str(merge_cfg.get("engine_file", "bevfusion_lidar.engine")) - - merged_main_body = { - "bevfusion_main_body": { - "onnx_file": onnx_file, - "engine_file": engine_file, - "io": { - "inputs": [{"name": inp.name, "dtype": inp.dtype} for inp in sparse_cfg.io.inputs], - "outputs": [{"name": out.name, "dtype": out.dtype} for out in dense_cfg.io.outputs], - "dynamic_axes": dict(sparse_cfg.io.dynamic_axes), - }, - "tensorrt_profile": { - name: { - "min_shape": list(profile.min_shape), - "opt_shape": list(profile.opt_shape), - "max_shape": list(profile.max_shape), - } - for name, profile in sparse_cfg.tensorrt_profile.items() - }, - } - } - - raw_components = {} - for name, comp in components_cfg.items(): - raw_components[name] = { - "onnx_file": comp.onnx_file, - "engine_file": comp.engine_file, - "io": { - "inputs": [{"name": inp.name, "dtype": inp.dtype} for inp in comp.io.inputs], - "outputs": [{"name": out.name, "dtype": out.dtype} for out in comp.io.outputs], - "dynamic_axes": dict(comp.io.dynamic_axes), - }, - "tensorrt_profile": { - k: { - "min_shape": list(v.min_shape), - "opt_shape": list(v.opt_shape), - "max_shape": list(v.max_shape), - } - for k, v in comp.tensorrt_profile.items() - }, - } - raw_components.update(merged_main_body) - return ComponentsConfig.from_dict(raw_components) diff --git a/deployment/projects/bevfusion/io/coors_contract.py b/deployment/projects/bevfusion/io/coors_contract.py deleted file mode 100644 index 7ad2ace90..000000000 --- a/deployment/projects/bevfusion/io/coors_contract.py +++ /dev/null @@ -1,29 +0,0 @@ -"""BEVFusion sparse ``coors`` layout for deploy / ONNX / TensorRT. - -Voxelization (``pts_voxel_layer``) returns indices as ``[x, y, z]`` (see -``dynamic_voxelize_kernel`` in ``bevfusion/ops/voxel``). - -Legacy Autoware-compatible ONNX expects graph **inputs** as ``[z, y, x]`` (no batch). -Inside the exported wrapper, indices are flipped to ``[x, y, z]`` and a batch column -is prepended before ``pts_middle_encoder`` (``sparse_shape`` is ``[H, W, D]``). - -PyTorch evaluation uses ``[batch, x, y, z]`` directly and does not use this module. -""" - -from __future__ import annotations - -import torch - - -def voxel_indices_xyz_to_graph_input_zyx(coors: torch.Tensor) -> torch.Tensor: - """``[M, 3]`` voxel indices ``[x, y, z]`` → graph input ``[z, y, x]``.""" - if coors.ndim != 2 or coors.shape[1] != 3: - return coors - return coors.flip(dims=[-1]).contiguous() - - -def graph_input_zyx_to_model_indices_xyz(coors: torch.Tensor) -> torch.Tensor: - """``[M, 3]`` graph input ``[z, y, x]`` → model indices ``[x, y, z]`` (wrapper flip).""" - if coors.ndim != 2 or coors.shape[1] != 3: - return coors - return coors.flip(dims=[-1]).contiguous() diff --git a/deployment/projects/bevfusion/io/data_loader.py b/deployment/projects/bevfusion/io/data_loader.py deleted file mode 100644 index 5139fb992..000000000 --- a/deployment/projects/bevfusion/io/data_loader.py +++ /dev/null @@ -1,82 +0,0 @@ -"""BEVFusion DataLoader for deployment. - -Wraps MMDet3D Dataset to load point cloud data for BEVFusion inference. -Pipeline runs once per sample in load_sample(), avoiding redundant computation. -""" - -from __future__ import annotations - -import copy -from typing import Dict, List, Optional, Union - -import torch -from mmengine.config import Config -from mmengine.registry import DATASETS, init_default_scope -from typing_extensions import override - -from deployment.io.base_data_loader import BaseDataLoader - - -class BEVFusionDataLoader(BaseDataLoader): - """Deployment dataloader for BEVFusion using MMDet3D Dataset. - - Wraps the same Dataset used by training/testing, ensuring identical - GT and pipeline processing. - """ - - def __init__(self, info_file: str, model_cfg: Config) -> None: - super().__init__() - self.model_cfg = model_cfg - self.info_file = info_file - self.dataset = self._build_dataset(model_cfg, info_file) - - def _build_dataset(self, model_cfg: Config, info_file: str) -> torch.utils.data.Dataset: - init_default_scope("mmdet3d") - if not hasattr(model_cfg, "test_dataloader"): - raise ValueError("model_cfg must have 'test_dataloader' with dataset config") - dataset_cfg = copy.deepcopy(model_cfg.test_dataloader.dataset) - dataset_cfg["ann_file"] = info_file - dataset_cfg["test_mode"] = True - return DATASETS.build(dataset_cfg) - - @override - def load_sample(self, index: int) -> Dict[str, Union[torch.Tensor, Dict[str, object]]]: - if index >= len(self.dataset): - raise IndexError(f"Sample index {index} out of range (0-{len(self.dataset)-1})") - - data = self.dataset[index] - pipeline_inputs = data["inputs"] - points_tensor = pipeline_inputs["points"].to("cpu") - - data_samples = data["data_samples"] - metainfo = getattr(data_samples, "metainfo", None) - eval_ann_info = getattr(data_samples, "eval_ann_info", None) - ground_truth = dict(eval_ann_info) if eval_ann_info else {} - - return { - "points": points_tensor, - "metainfo": dict(metainfo) if metainfo else {}, - "ground_truth": ground_truth, - } - - @override - def preprocess( - self, sample: Dict[str, Union[torch.Tensor, Dict[str, object]]] - ) -> Dict[str, Union[torch.Tensor, Dict[str, object]]]: - return { - "points": sample["points"], - "metainfo": sample["metainfo"], - } - - @property - @override - def num_samples(self) -> int: - return len(self.dataset) - - @property - def class_names(self) -> List[str]: - if hasattr(self.dataset, "metainfo") and "classes" in self.dataset.metainfo: - return list(self.dataset.metainfo["classes"]) - if hasattr(self.model_cfg, "class_names"): - return list(self.model_cfg.class_names) - raise ValueError("class_names not found in dataset.metainfo or model_cfg") diff --git a/deployment/projects/bevfusion/io/model_loader.py b/deployment/projects/bevfusion/io/model_loader.py deleted file mode 100644 index b995009bf..000000000 --- a/deployment/projects/bevfusion/io/model_loader.py +++ /dev/null @@ -1,65 +0,0 @@ -"""BEVFusion model loading utilities for deployment.""" - -from __future__ import annotations - -import copy -import logging - -import torch -from mmengine.config import Config -from mmengine.registry import MODELS, init_default_scope -from mmengine.runner import load_checkpoint - -from deployment.primitives.device import DeviceSpec - -logger = logging.getLogger(__name__) - - -def _register_bevfusion_modules() -> None: - """Register BEVFusion and SparseConvolution modules into MMDet3D registries.""" - import projects.BEVFusion.bevfusion # noqa: F401 - import projects.SparseConvolution # noqa: F401 - - -def build_bevfusion_model( - model_cfg: Config, - checkpoint_path: str, - device: DeviceSpec, - *, - fuse_spconv_bn: bool = False, -) -> torch.nn.Module: - """Build a BEVFusion model from config and load checkpoint weights. - - Args: - model_cfg: MMEngine model configuration. - checkpoint_path: Path to .pth checkpoint file. - device: Target device. - fuse_spconv_bn: If True, fuse each ``SparseConvolution`` + ``BatchNorm1d`` pair in - ``pts_middle_encoder`` after ``load_checkpoint`` (eval-mode Conv-BN fold, a graph - optimization for the sparse ONNX export). - - Returns: - Loaded and eval-mode BEVFusion model. - """ - init_default_scope("mmdet3d") - _register_bevfusion_modules() - - model_config = copy.deepcopy(model_cfg.model) - model = MODELS.build(model_config) - - torch_device = device.to_torch_device() - model.to(torch_device) - - load_checkpoint(model, checkpoint_path, map_location=torch_device) - - if fuse_spconv_bn: - from deployment.projects.bevfusion.export.spconv_bn_fusion import fuse_spconv_bn_in_encoder - - encoder = getattr(model, "pts_middle_encoder", None) - if encoder is not None: - count = fuse_spconv_bn_in_encoder(encoder) - logger.info("Fused %d SparseConv-BN pair(s) in pts_middle_encoder", count) - - model.eval() - model.cfg = model_cfg - return model diff --git a/deployment/projects/bevfusion/runner.py b/deployment/projects/bevfusion/runner.py deleted file mode 100644 index 05a8ca6c0..000000000 --- a/deployment/projects/bevfusion/runner.py +++ /dev/null @@ -1,92 +0,0 @@ -"""BEVFusion-specific deployment runner.""" - -from __future__ import annotations - -import logging -from typing import Optional, Tuple - -import torch -from mmengine.config import Config - -from deployment.config.base import BaseDeploymentConfig -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.export.contexts import ExportContext -from deployment.io.base_data_loader import BaseDataLoader -from deployment.projects.bevfusion.evaluation.evaluator import BEVFusionEvaluator -from deployment.projects.bevfusion.export.onnx_export_pipeline import BEVFusionONNXExportPipeline -from deployment.projects.bevfusion.export.tensorrt_export_pipeline import BEVFusionTensorRTExportPipeline -from deployment.projects.bevfusion.io.model_loader import build_bevfusion_model -from deployment.runtime.runner import BaseDeploymentRunner - -logger = logging.getLogger(__name__) - - -class BEVFusionDeploymentRunner(BaseDeploymentRunner): - """BEVFusion deployment runner. - - Constructs BEVFusion's model-specific ONNX/TensorRT export pipelines and injects them - into the project-agnostic ``BaseDeploymentRunner`` via its ``onnx_pipeline`` / - ``tensorrt_pipeline`` override hooks (BEVFusion needs wrapper modules, TopK constant - folding, coordinate flips, and split→merge ONNX composition that the generic whole-model - export cannot express). - - BEVFusion-only deploy-config flags (``fuse_spconv_bn``, ``spconv_do_sort``, - ``spconv_fuse_implicit_gemm_relu``) are read from the raw ``deploy_cfg`` passed in by the - entrypoint, since ``BaseDeploymentConfig`` only surfaces typed sections. - """ - - def __init__( - self, - data_loader: BaseDataLoader, - evaluator: BEVFusionEvaluator, - executor: BackendExecutor, - config: BaseDeploymentConfig, - model_cfg: Config, - deploy_cfg: Config, - module: str = "main_body", - plugin_libraries: Tuple[str, ...] = (), - onnx_pipeline: Optional[BEVFusionONNXExportPipeline] = None, - tensorrt_pipeline: Optional[BEVFusionTensorRTExportPipeline] = None, - ) -> None: - self._module = module - self._deploy_cfg = deploy_cfg - - # Construct the model-specific pipelines BEFORE super().__init__, because the base - # runner forwards them straight to the ExportOrchestrator (there is no post-init slot). - if onnx_pipeline is None: - onnx_pipeline = BEVFusionONNXExportPipeline(module=module) - if tensorrt_pipeline is None: - tensorrt_pipeline = BEVFusionTensorRTExportPipeline( - components_cfg=config.components_cfg, - plugin_libraries=tuple(plugin_libraries), - ) - - super().__init__( - data_loader=data_loader, - evaluator=evaluator, - executor=executor, - config=config, - model_cfg=model_cfg, - onnx_pipeline=onnx_pipeline, - tensorrt_pipeline=tensorrt_pipeline, - ) - - def load_pytorch_model(self, checkpoint_path: str, context: ExportContext) -> torch.nn.Module: - """Load the BEVFusion model onto the CUDA device for export. - - The base runner forwards the returned model to ``executor.set_pytorch_model`` after - export, so PyTorch/ONNX/TensorRT evaluation all reuse this reference. - """ - cuda_device = self.config.device_config.cuda - if cuda_device is None: - raise RuntimeError( - "BEVFusion requires a CUDA device for sparse convolution. Set devices.cuda in deploy config." - ) - - fuse_spconv_bn = bool(self._deploy_cfg.get("fuse_spconv_bn", False)) - return build_bevfusion_model( - model_cfg=self.model_cfg, - checkpoint_path=checkpoint_path, - device=cuda_device, - fuse_spconv_bn=fuse_spconv_bn, - ) diff --git a/deployment/projects/bevfusion/README.md b/deployment/projects/bevfusion_l/README.md similarity index 73% rename from deployment/projects/bevfusion/README.md rename to deployment/projects/bevfusion_l/README.md index a7e56c106..30ced91a6 100644 --- a/deployment/projects/bevfusion/README.md +++ b/deployment/projects/bevfusion_l/README.md @@ -13,9 +13,9 @@ first — BEVFusion is one *project bundle* that implements the stage contract d ```mermaid flowchart TD - ckpt["FP32/FP16 checkpoint"] --> run["deployment.cli.main bevfusion"] + ckpt["FP32/FP16 checkpoint"] --> run["deployment.cli.main bevfusion_l"] run --> load["model_loader
build model, load_checkpoint"] - load --> onnx["ONNX export
sparse.onnx + dense.onnx (or single main_body)"] + load --> onnx["ONNX export
sparse.onnx + dense.onnx (→ merged full graph)"] onnx --> trt["TensorRT engines
+ Autoware ImplicitGemm plugin (sparse)"] trt --> eval["evaluate / verify
PyTorch vs ONNX vs TRT"] ``` @@ -23,22 +23,24 @@ flowchart TD Single entry point: ```bash -python -m deployment.cli.main bevfusion --module main_body +python -m deployment.cli.main bevfusion_l ``` --- -## 2. BEVFusion project bundle (`deployment/projects/bevfusion/`) +## 2. BEVFusion project bundle (`deployment/projects/bevfusion_l/`) Mirrors the framework stage contract. Wiring: [`entrypoint.py:run`](entrypoint.py) builds config + data loader + executor + evaluator, then [`runner.py:BEVFusionDeploymentRunner`](runner.py) -(a thin `BaseDeploymentRunner`) injects BEVFusion's ONNX/TensorRT export pipelines. +(a thin `BaseDeploymentRunner`) drives the shared `OnnxExportPipeline` by injecting BEVFusion's +`BEVFusionSampleExtractor` + `BEVFusionComponentBuilder` (the same seam pattern CenterPoint uses), +and reuses the shared `TensorRTExportPipeline`. | Stage | Directory | Key modules | | --- | --- | --- | -| Config | [`config/`](config/) | `deploy_config.py` (single) + `deploy_config_split_fp16_opt_2_8.py` (split) (§4) | +| Config | [`config/`](config/) | `deploy_config.py` (split, optimized) + `deploy_config_without_opt.py` (split, no opt) (§4) | | IO | [`io/`](io/) | [`model_loader.py`](io/model_loader.py) (build + `load_checkpoint`, optional sparse BN fuse), `data_loader.py`, `coors_contract.py` (voxel `[x,y,z]`→graph `[z,y,x]`), `component_utils.py` (split vs merged) | -| Export | [`export/`](export/) | [`onnx_export_pipeline.py`](export/onnx_export_pipeline.py) (sparse/dense/main_body wrappers, TopK fix, float shadow, ImplicitGemm ReLU fuse), `spconv_bn_fusion.py`, `sparse_encoder_float_shadow.py`, `onnx_fuse_implicit_gemm_activation.py`, `tensorrt_export_pipeline.py` | +| Export | [`export/`](export/) | [`sample_extractor.py`](export/sample_extractor.py) (voxelize) + [`component_builder.py`](export/component_builder.py) (split sparse/dense, wires trace-context + post-transforms) feed the shared `OnnxExportPipeline`; `onnx_models/bevfusion_onnx.py` (sparse/dense wrappers), `transforms.py` (TopK fix, split→merge), `sparse_encoder_float_shadow.py` (FP32 shadow + trace context), `onnx_fuse_implicit_gemm_activation.py` (ImplicitGemm ReLU fuse), `spconv_bn_fusion.py` | | Inference | [`inference/`](inference/) | `pytorch_/onnx_/tensorrt_inference_pipeline.py` (all `preprocess→run→postprocess`) | | Evaluation | [`evaluation/`](evaluation/) | `executor.py` (pipeline construction + output routing), `evaluator.py` (3D metrics + latency breakdown) | @@ -46,8 +48,8 @@ data loader + executor + evaluator, then [`runner.py:BEVFusionDeploymentRunner`] ## 3. Sparse vs dense split -BEVFusion (LiDAR) exports as two components so the dense tower can go to plain TensorRT while the -sparse tower uses the custom `ImplicitGemm` plugin: +BEVFusion (LiDAR) exports as two components so the dense backbone/neck/head can go to plain +TensorRT while the sparse encoder uses the custom `ImplicitGemm` plugin: | | Sparse encoder (`pts_middle_encoder`) | Dense backbone/neck/head | | --- | --- | --- | @@ -55,7 +57,7 @@ sparse tower uses the custom `ImplicitGemm` plugin: | ONNX op | `autoware::ImplicitGemm` (custom) | standard `Conv2d/ReLU/Add` | | TensorRT | **custom plugin** `ImplicitGemm` (`libautoware_tensorrt_plugins.so`) | TRT-native | -The sparse tower is traced through a **fused FP32 shadow encoder** +The sparse encoder is traced through a **fused FP32 shadow encoder** ([`sparse_encoder_float_shadow.py`](export/sparse_encoder_float_shadow.py)) so BN can be folded (`fuse_spconv_bn`) into a clean BN-free sparse ONNX without mutating the runtime model. Graph knobs: @@ -76,8 +78,8 @@ All configs are MMEngine files. | Config | Topology | Precision | | --- | --- | --- | -| [`deploy_config.py`](config/deploy_config.py) | single `main_body` | FP32/FP16 | -| [`deploy_config_split_fp16_opt_2_8.py`](config/deploy_config_split_fp16_opt_2_8.py) | split sparse+dense | FP16 (optimized) | +| [`deploy_config.py`](config/deploy_config.py) | split sparse+dense (+merge) | FP16 (optimized) | +| [`deploy_config_without_opt.py`](config/deploy_config_without_opt.py) | split sparse+dense (+merge) | FP16 (no opt) | **Isolation tip:** to check whether the split/voxel/eval pipeline is healthy, keep the same CLI/config/work_dir and point `checkpoint_path` at the FP32 `.pth`. If mAP is fine, the pipeline is diff --git a/deployment/projects/bevfusion/__init__.py b/deployment/projects/bevfusion_l/__init__.py similarity index 70% rename from deployment/projects/bevfusion/__init__.py rename to deployment/projects/bevfusion_l/__init__.py index 8d3942a29..415a93135 100644 --- a/deployment/projects/bevfusion/__init__.py +++ b/deployment/projects/bevfusion_l/__init__.py @@ -6,14 +6,12 @@ from __future__ import annotations -from deployment.projects.bevfusion.cli import add_args -from deployment.projects.bevfusion.entrypoint import run +from deployment.projects.bevfusion_l.entrypoint import run from deployment.projects.registry import ProjectAdapter, project_registry project_registry.register( ProjectAdapter( - name="bevfusion", - add_args=add_args, + name="bevfusion_l", run=run, ) ) diff --git a/deployment/projects/bevfusion_l/config/bevfusion_deployment_config.py b/deployment/projects/bevfusion_l/config/bevfusion_deployment_config.py new file mode 100644 index 000000000..7a5f46bc4 --- /dev/null +++ b/deployment/projects/bevfusion_l/config/bevfusion_deployment_config.py @@ -0,0 +1,64 @@ +"""BEVFusion-specific deployment config. + +Extends :class:`~deployment.config.base.BaseDeploymentConfig` to model the BEVFusion-only +deploy-config keys as typed attributes, so the entrypoint and export pipeline never reach +back into the raw MMEngine ``Config``. This is the typed home for the keys the generic +sections intentionally do not model. +""" + +from __future__ import annotations + +from mmengine.config import Config + +from deployment.config.base import BaseDeploymentConfig +from deployment.projects.bevfusion_l.config.component_layout import ( + add_merged_component, + is_split_components, + merge_requested, +) + + +class BEVFusionDeploymentConfig(BaseDeploymentConfig): + """Deployment config for BEVFusion. + + Adds typed attributes for the BEVFusion-only deploy-config keys: + + - ``fuse_spconv_bn``: fold spconv BatchNorm into conv weights before export (default ``False``). + - ``spconv_do_sort``: bake the pair-mask argsort into ``GetIndicePairsImplicitGemm`` at ONNX + export (default ``True``). + - ``spconv_fuse_implicit_gemm_relu``: fuse a trailing ReLU into ImplicitGemm nodes in the sparse + ONNX postprocess (default ``False``). + - ``merge_bevfusion``: keep the split (sparse+dense) export and also emit the merged + full-graph artifacts (derived from the deploy config's ``bevfusion_merge`` key). + """ + + def __init__(self, deploy_cfg: Config) -> None: + super().__init__(deploy_cfg) + self.fuse_spconv_bn: bool = bool(deploy_cfg.get("fuse_spconv_bn", False)) + self.spconv_do_sort: bool = bool(deploy_cfg.get("spconv_do_sort", True)) + self.spconv_fuse_implicit_gemm_relu: bool = bool(deploy_cfg.get("spconv_fuse_implicit_gemm_relu", False)) + self.merge_bevfusion: bool = merge_requested(deploy_cfg) + + # The merged graph is *derived* from the split sparse+dense pair (sparse inputs + + # dense outputs), so it is resolved here as part of the config rather than mutated onto + # the config later. After construction ``components_cfg`` is the final layout: the split + # (sparse+dense) export plus, when ``merge_bevfusion`` is set, the merged graph. + if self.merge_bevfusion: + self.components_cfg = add_merged_component( + deploy_cfg=deploy_cfg, + components_cfg=self.components_cfg, + ) + + self._validate_components() + + def _validate_components(self) -> None: + """Fail early if the resolved component layout is incomplete. + + BEVFusion's required components vary by layout (split sparse+dense vs merged graph), + so this layout-aware check is authoritative rather than the registry's static tuple. + """ + if is_split_components(self.components_cfg): + self.components_cfg.get_component("bevfusion_sparse") + self.components_cfg.get_component("bevfusion_dense") + else: + self.components_cfg.get_component("bevfusion_merged") diff --git a/deployment/projects/bevfusion_l/config/component_layout.py b/deployment/projects/bevfusion_l/config/component_layout.py new file mode 100644 index 000000000..4d9b1df7e --- /dev/null +++ b/deployment/projects/bevfusion_l/config/component_layout.py @@ -0,0 +1,92 @@ +"""BEVFusion deploy-config component-layout helpers. + +BEVFusion can be deployed in two component layouts: + +- **split**: separate ``bevfusion_sparse`` (spconv) + ``bevfusion_dense`` ONNX/TensorRT graphs. +- **merged**: a single ``bevfusion_merged`` graph (sparse inputs → dense outputs). + +These helpers query the layout (:func:`is_split_components`, :func:`has_component`) and, when a +deploy config opts into ``bevfusion_merge``, derive the merged ``bevfusion_merged`` component +from the split pair (:func:`merge_requested`, :func:`add_merged_component`). They operate +purely on deploy-config structures, so they live beside the deployment config rather than in ``io``. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from deployment.config.schema import ComponentCfg, ComponentIO, ComponentsConfig + + +def is_split_components(components_cfg: ComponentsConfig) -> bool: + """True when the deploy config uses split sparse + dense graphs (not a single merged graph).""" + names = set(components_cfg.component_names()) + return "bevfusion_sparse" in names and "bevfusion_dense" in names + + +def has_component(components_cfg: ComponentsConfig, component_name: str) -> bool: + """Return True if the component exists in the layout.""" + try: + components_cfg.get_component(component_name) + return True + except KeyError: + return False + + +def merge_requested(deploy_cfg: Mapping[str, Any]) -> bool: + """Return True when the deploy config requests the split->merged graph merge. + + The single canonical key is ``bevfusion_merge`` (a dict with ``enabled`` / ``onnx_file`` / + ``engine_file``, or a plain bool). + """ + merge_raw = deploy_cfg.get("bevfusion_merge", False) + if isinstance(merge_raw, Mapping): + return bool(merge_raw.get("enabled", False)) + return bool(merge_raw) + + +def add_merged_component( + *, + deploy_cfg: Mapping[str, Any], + components_cfg: ComponentsConfig, +) -> ComponentsConfig: + """Add a merged ``bevfusion_merged`` component while keeping the split components. + + When ``bevfusion_merge`` is enabled and the layout is split, this derives + ``bevfusion_merged`` by reusing: + + - the split sparse input schema / TensorRT profile, and + - the split dense output schema. + + If the layout is not split, merge is not requested, or ``bevfusion_merged`` already exists, + the config is returned unchanged. + """ + if not is_split_components(components_cfg): + return components_cfg + if not merge_requested(deploy_cfg): + return components_cfg + if has_component(components_cfg, "bevfusion_merged"): + return components_cfg + + sparse_cfg = components_cfg.get_component("bevfusion_sparse") + dense_cfg = components_cfg.get_component("bevfusion_dense") + + merge_raw = deploy_cfg.get("bevfusion_merge", {}) + merge_cfg = merge_raw if isinstance(merge_raw, Mapping) else {} + onnx_file = str(merge_cfg.get("onnx_file", "bevfusion_lidar.onnx")) + engine_file = str(merge_cfg.get("engine_file", "bevfusion_lidar.engine")) + + # The merged graph reuses the split sparse inputs / TensorRT profile and the split dense + # outputs, so build it directly from the already-typed components (no raw-dict round-trip). + merged = ComponentCfg( + name="bevfusion_merged", + onnx_file=onnx_file, + engine_file=engine_file, + io=ComponentIO( + inputs=list(sparse_cfg.io.inputs), + outputs=list(dense_cfg.io.outputs), + dynamic_axes=dict(sparse_cfg.io.dynamic_axes), + ), + tensorrt_profile=dict(sparse_cfg.tensorrt_profile), + ) + return components_cfg.with_component(merged) diff --git a/deployment/projects/bevfusion_l/config/deploy_config.py b/deployment/projects/bevfusion_l/config/deploy_config.py new file mode 100644 index 000000000..a6fada3fc --- /dev/null +++ b/deployment/projects/bevfusion_l/config/deploy_config.py @@ -0,0 +1,220 @@ +""" +BEVFusion-L Deployment Configuration — split sparse+dense, merged & optimized (FP16). + +Layout (single file, grouped by concern; mirrors centerpoint/config/deploy_config.py): + 1. SHARED VALUES - single source of truth reused across sections (paths, devices, shapes). + 2. EXPORT - spconv/export flags, ONNX/TensorRT build settings, component definitions. + 3. EVALUATION - per-backend evaluation settings. + 4. VERIFICATION - cross-backend numerical verification scenarios. + +Only the top-level names `checkpoint_path`, `devices`, `export`, `components`, `runtime_io`, +`onnx_config`, `tensorrt_config`, `evaluation`, `verification`, plus the BEVFusion-only flags +`spconv_do_sort`, `spconv_fuse_implicit_gemm_relu`, `fuse_spconv_bn`, `bevfusion_merge`, are read +(by `BaseDeploymentConfig` / `BEVFusionDeploymentConfig`). Names prefixed with `_` are local +single-source helpers and are intentionally not consumed directly. + +Requirements: + - LiDAR-only model (`fusion_layer is None`, `img_backbone is None`). + - `components.bevfusion_dense.tensorrt_profile.lidar_bev` H,W must equal + `grid_size[0:2] // out_size_factor` (e.g. 1440/8 → 180×180). Do NOT widen H/W: `bbox_head` + uses fixed `bev_pos` and heatmap length H*W, so a wide profile breaks Reshape/Gather and + yields garbage mAP. Adjust channel C (default 256) to the sparse encoder output. + +CLI:: + + python -m deployment.cli.main bevfusion_l \\ + deployment/projects/bevfusion_l/config/deploy_config.py \\ + +""" + +# ============================================================================ +# 1. SHARED VALUES (single source of truth) +# ============================================================================ + +# Checkpoint - single source of truth for the PyTorch model (used by export + PyTorch eval). +checkpoint_path = "work_dirs/bevfusion/bevfusion_2_8/best_epoch_25.pth" +# checkpoint_path = "vivid/bench_comparison/bevfusion_2_7/best_epoch_28.pth" + +# Device settings (shared by export, evaluation, verification). +devices = dict( + cpu="cpu", + cuda="cuda:0", +) +# Alias reused by the per-backend evaluation settings below so the CUDA device is written once. +_CUDA = devices["cuda"] + +# Deployment output layout. _ONNX_DIR / _TENSORRT_DIR are the single source for both the export +# outputs and the evaluation backends' engine_dir (kept in sync here). +_WORK_DIR = "work_dirs/bevfusion_deployment_2_8" +_ONNX_DIR = f"{_WORK_DIR}/onnx" +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# Dense head BEV feature map: [batch, channels, grid_h, grid_w] = grid_size // out_size_factor. +# Fully static (min == opt == max) so constant folding can drop the head's shape-glue (~177 +# nodes) and the split graph's node count matches the monolithic export. +_LIDAR_BEV_SHAPE = [1, 256, 180, 180] + +# Sparse (spconv) voxel input profile: [num_voxels, max_points_per_voxel, voxel_feature_dim]. +_MAX_POINTS_PER_VOXEL = 32 +_VOXEL_FEATURE_DIM = 5 +_VOXELS_OPT = 64000 +_VOXELS_MAX = 256000 + +# ============================================================================ +# 2. EXPORT +# ============================================================================ + +# Bake the pair-mask argsort into GetIndicePairsImplicitGemm.do_sort_i at ONNX symbolic export. +spconv_do_sort = False + +# Sparse ONNX postprocess: fuse each `ImplicitGemm -> Relu` into an activated ImplicitGemm +# (applied as a post-export transform by export/component_builder.py after exporting the sparse ONNX). +# - True : bake activation into ImplicitGemm (act_type) +# - False : keep explicit Relu nodes +spconv_fuse_implicit_gemm_relu = True + +# Fuse SparseConv + BN in `pts_middle_encoder` before ONNX export (eval-mode Conv-BN fold), +# producing a BN-free sparse subgraph in the exported ONNX. +fuse_spconv_bn = True + +# Export mode: "onnx", "trt", "both", "none". sample_idx: dataset index used to trace/shape. +export = dict( + mode="none", + work_dir=_WORK_DIR, + onnx_path=_ONNX_DIR, + sample_idx=0, +) + +# Keep the split component definitions but also emit a single merged full-graph ONNX/engine. +# - enabled=False: split sparse+dense ONNX/engine only. +# - enabled=True : additionally emit one merged ONNX + engine + backend pipeline. +bevfusion_merge = dict( + enabled=True, + onnx_file="bevfusion_lidar_fp16_opt.onnx", + engine_file="bevfusion_lidar_fp16_opt.engine", +) + +# ONNX export settings (shared across all components). +# BEVFusion 2.8.x exports at opset 18 (matches projects/BEVFusion/configs/deploy/*_tensorrt_dynamic.py). +onnx_config = dict( + opset_version=18, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +# TensorRT build settings (shared across all components). plugin_libraries loads the spconv +# ImplicitGemm plugin before engine build/deserialize. +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=1 << 32, + plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"], +) + +# Split components (must keep keys `bevfusion_sparse` + `bevfusion_dense`; the merged full graph +# is derived from this pair by BEVFusionDeploymentConfig when bevfusion_merge is enabled). +components = dict( + bevfusion_sparse=dict( + onnx_file="bevfusion_sparse.onnx", + engine_file="bevfusion_sparse.engine", + io=dict( + inputs=[ + dict(name="voxels", dtype="float32"), + dict(name="coors", dtype="int32"), + dict(name="num_points_per_voxel", dtype="int32"), + ], + outputs=[ + dict(name="lidar_bev", dtype="float32"), + ], + dynamic_axes={ + "voxels": {0: "voxels_num"}, + "coors": {0: "voxels_num"}, + "num_points_per_voxel": {0: "voxels_num"}, + }, + ), + tensorrt_profile=dict( + voxels=dict( + min_shape=[1, _MAX_POINTS_PER_VOXEL, _VOXEL_FEATURE_DIM], + opt_shape=[_VOXELS_OPT, _MAX_POINTS_PER_VOXEL, _VOXEL_FEATURE_DIM], + max_shape=[_VOXELS_MAX, _MAX_POINTS_PER_VOXEL, _VOXEL_FEATURE_DIM], + ), + coors=dict( + min_shape=[1, 3], + opt_shape=[_VOXELS_OPT, 3], + max_shape=[_VOXELS_MAX, 3], + ), + num_points_per_voxel=dict( + min_shape=[1], + opt_shape=[_VOXELS_OPT], + max_shape=[_VOXELS_MAX], + ), + ), + ), + bevfusion_dense=dict( + onnx_file="bevfusion_dense.onnx", + engine_file="bevfusion_dense.engine", + io=dict( + inputs=[ + dict(name="lidar_bev", dtype="float32"), + ], + outputs=[ + dict(name="bbox_pred", dtype="float32"), + dict(name="score", dtype="float32"), + dict(name="label_pred", dtype="int64"), + ], + # Static lidar_bev input (see _LIDAR_BEV_SHAPE), so no dynamic axes. + dynamic_axes={}, + ), + # H,W fixed to head grid (grid_size // out_size_factor). Widen only batch dim if needed. + tensorrt_profile=dict( + lidar_bev=dict( + min_shape=_LIDAR_BEV_SHAPE, + opt_shape=_LIDAR_BEV_SHAPE, + max_shape=_LIDAR_BEV_SHAPE, + ), + ), + ), +) + +runtime_io = dict( + info_file="info/t4dataset_j6gen2_base_infos_test.pkl", +) + +# ============================================================================ +# 3. EVALUATION +# ONNX *inference* is unsupported for BEVFusion (sparse graph needs the TRT plugin). +# ============================================================================ +evaluation = dict( + enabled=True, + num_samples=5, + num_warmup=2, + verbose=True, + backends=dict( + pytorch=dict( + enabled=False, + device=_CUDA, + ), + tensorrt=dict( + enabled=True, + device=_CUDA, + engine_dir=_TENSORRT_DIR, + ), + ), +) + +# ============================================================================ +# 4. VERIFICATION +# ============================================================================ +verification = dict( + enabled=False, + tolerance=1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[], + onnx=[], + trt=[], + none=[], + ), +) diff --git a/deployment/projects/bevfusion_l/config/deploy_config_original.py b/deployment/projects/bevfusion_l/config/deploy_config_original.py new file mode 100644 index 000000000..a537f024b --- /dev/null +++ b/deployment/projects/bevfusion_l/config/deploy_config_original.py @@ -0,0 +1,198 @@ +""" +BEVFusion deploy config — **split ONNX / TensorRT (route 1)** + +Use this instead of ``deploy_config.py`` when you want: + - ``bevfusion_sparse.onnx`` / ``.engine`` — voxelization stays outside; graph is + ``pts_middle_encoder`` only (spconv / plugins / libspconv). + - ``bevfusion_dense.onnx`` / ``.engine`` — ``pts_backbone`` + ``pts_neck`` + ``bbox_head`` + (+ the head postprocess). Suitable for plain TensorRT without spconv ops. + +**Requirements** + - LiDAR-only model: ``fusion_layer is None`` and ``img_backbone is None``. + + - Set ``bevfusion_dense.tensorrt_profile.lidar_bev`` **H,W** to ``grid_size[0:2] // out_size_factor`` + (e.g. 1440/8 → **180×180**). Do **not** use a wide H/W range: ``bbox_head`` uses fixed ``bev_pos`` + and heatmap length ``H*W``; TRT profiles like 32×32 or 2048×2048 break ``Reshape``/``Gather`` + consistency and yield garbage mAP. + - Adjust channel **C** (default ``256``) to match ``pts_backbone.in_channels`` / sparse encoder output. + +CLI:: + + python -m deployment.cli.main bevfusion_l \\ + deployment/projects/bevfusion_l/config/deploy_config_without_opt.py \\ + +""" + +spconv_do_sort = False + +# ============================================================================ +# Sparse ONNX postprocess (FP): fuse ImplicitGemm with trailing Relu/Add(const)+Relu. +# ---------------------------------------------------------------------------- +# Applied automatically as a post-export transform by +# deployment/projects/bevfusion_l/export/component_builder.py after exporting ``bevfusion_sparse.onnx``. +# - True : bake activation into ImplicitGemm (act_type / optional 6th bias input) +# - False : keep explicit Relu/Add nodes +# ============================================================================ +spconv_fuse_implicit_gemm_relu = False + +# Fuse SparseConv + BN in ``pts_middle_encoder`` before ONNX export (eval-mode Conv-BN fold). +# Produces a BN-free sparse subgraph in the exported ONNX. +fuse_spconv_bn = False + +# ============================================================================ +# Checkpoint Path +# ============================================================================ +checkpoint_path = "work_dirs/bevfusion/bevfusion_2_8/best_epoch_25.pth" +# checkpoint_path = "vivid/bench_comparison/bevfusion_2_7/best_epoch_28.pth" + + +devices = dict( + cpu="cpu", + cuda="cuda:0", +) + +export = dict( + mode="trt", + work_dir="work_dirs/bevfusion_deployment_2_8_original", + onnx_path="work_dirs/bevfusion_deployment_2_8_original/onnx", + # Dataset index of the sample used to trace/shape the exported model (read by ExportConfig, + # same as CenterPoint's export.sample_idx). + sample_idx=0, +) + + +# Optional: keep split component definitions but also emit one merged full-graph ONNX/engine. +# - False: split sparse+dense ONNX/engine (default) +# - True : one ONNX + one engine + one backend pipeline +bevfusion_merge = dict( + enabled=True, + onnx_file="bevfusion_lidar.onnx", + engine_file="bevfusion_lidar.engine", +) + + +_WORK_DIR = str(export["work_dir"]).rstrip("/") +_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" + +# ============================================================================ +# Split components (must keep keys ``bevfusion_sparse`` + ``bevfusion_dense``) +# ============================================================================ +components = dict( + bevfusion_sparse=dict( + onnx_file="bevfusion_sparse.onnx", + engine_file="bevfusion_sparse.engine", + io=dict( + inputs=[ + dict(name="voxels", dtype="float32"), + dict(name="coors", dtype="int32"), + dict(name="num_points_per_voxel", dtype="int32"), + ], + outputs=[ + dict(name="lidar_bev", dtype="float32"), + ], + dynamic_axes={ + "voxels": {0: "voxels_num"}, + "coors": {0: "voxels_num"}, + "num_points_per_voxel": {0: "voxels_num"}, + # lidar_bev output is a fixed dense grid [1, C*D, 180, 180] regardless of voxel + # count; marking its batch/H/W dynamic re-dynamizes the boundary and defeats + # constant folding (adds ~50 shape-glue nodes). Keep it static. + }, + ), + tensorrt_profile=dict( + voxels=dict( + min_shape=[1, 32, 5], + opt_shape=[64000, 32, 5], + max_shape=[256000, 32, 5], + ), + coors=dict( + min_shape=[1, 3], + opt_shape=[64000, 3], + max_shape=[256000, 3], + ), + num_points_per_voxel=dict( + min_shape=[1], + opt_shape=[64000], + max_shape=[256000], + ), + ), + ), + bevfusion_dense=dict( + onnx_file="bevfusion_dense.onnx", + engine_file="bevfusion_dense.engine", + io=dict( + inputs=[ + dict(name="lidar_bev", dtype="float32"), + ], + outputs=[ + dict(name="bbox_pred", dtype="float32"), + dict(name="score", dtype="float32"), + dict(name="label_pred", dtype="int64"), + ], + # Spatial dims must stay at head BEV resolution; see module docstring. + # lidar_bev input is fully static ([1,256,180,180]; TRT profile is min=opt=max), + # so no dynamic axes — this lets constant folding remove the head's shape-glue + # (~177 nodes) and aligns the split graph node count with the monolithic export. + dynamic_axes={}, + ), + # H,W fixed to head grid (grid_size // out_size_factor). Widen only batch dim if needed. + tensorrt_profile=dict( + lidar_bev=dict( + min_shape=[1, 256, 180, 180], + opt_shape=[1, 256, 180, 180], + max_shape=[1, 256, 180, 180], + ), + ), + ), +) + +runtime_io = dict( + info_file="info/t4dataset_j6gen2_base_infos_test.pkl", +) + +onnx_config = dict( + # BEVFusion 2.8.x exports at opset 18 (matches projects/BEVFusion/configs/deploy/*_tensorrt_dynamic.py). + opset_version=18, + do_constant_folding=True, + export_params=True, + keep_initializers_as_inputs=False, + simplify=False, +) + +tensorrt_config = dict( + precision_policy="fp16", + max_workspace_size=1 << 32, + plugin_libraries=["/opt/plugins/libautoware_tensorrt_plugins.so"], +) + +evaluation = dict( + enabled=True, + num_samples=5, + num_warmup=2, + verbose=True, + # ONNX *inference* is unsupported for BEVFusion. + backends=dict( + pytorch=dict( + enabled=False, + device=devices["cuda"], + ), + tensorrt=dict( + enabled=True, + device=devices["cuda"], + engine_dir=_TENSORRT_DIR, + ), + ), +) + +verification = dict( + enabled=False, + tolerance=1, + num_verify_samples=1, + devices=devices, + scenarios=dict( + both=[], + onnx=[], + trt=[], + none=[], + ), +) diff --git a/deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py b/deployment/projects/bevfusion_l/config/deploy_config_without_opt.py similarity index 77% rename from deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py rename to deployment/projects/bevfusion_l/config/deploy_config_without_opt.py index 3d7167ac8..030337fd4 100644 --- a/deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py +++ b/deployment/projects/bevfusion_l/config/deploy_config_without_opt.py @@ -5,7 +5,7 @@ - ``bevfusion_sparse.onnx`` / ``.engine`` — voxelization stays outside; graph is ``pts_middle_encoder`` only (spconv / plugins / libspconv). - ``bevfusion_dense.onnx`` / ``.engine`` — ``pts_backbone`` + ``pts_neck`` + ``bbox_head`` - (+ same postprocess as single-file export). Suitable for plain TensorRT without spconv ops. + (+ the head postprocess). Suitable for plain TensorRT without spconv ops. **Requirements** - LiDAR-only model: ``fusion_layer is None`` and ``img_backbone is None``. @@ -14,12 +14,12 @@ (e.g. 1440/8 → **180×180**). Do **not** use a wide H/W range: ``bbox_head`` uses fixed ``bev_pos`` and heatmap length ``H*W``; TRT profiles like 32×32 or 2048×2048 break ``Reshape``/``Gather`` consistency and yield garbage mAP. - - Adjust channel **C** (default ``256``) to match ``pts_backbone.in_channels`` / sparse tower output. + - Adjust channel **C** (default ``256``) to match ``pts_backbone.in_channels`` / sparse encoder output. CLI:: - python -m deployment.cli.main bevfusion \\ - deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py \\ + python -m deployment.cli.main bevfusion_l \\ + deployment/projects/bevfusion_l/config/deploy_config_without_opt.py \\ """ @@ -28,16 +28,16 @@ # ============================================================================ # Sparse ONNX postprocess (FP): fuse ImplicitGemm with trailing Relu/Add(const)+Relu. # ---------------------------------------------------------------------------- -# Applied automatically by deployment/projects/bevfusion/export/onnx_export_pipeline.py -# after exporting ``bevfusion_sparse.onnx``. +# Applied automatically as a post-export transform by +# deployment/projects/bevfusion_l/export/component_builder.py after exporting ``bevfusion_sparse.onnx``. # - True : bake activation into ImplicitGemm (act_type / optional 6th bias input) # - False : keep explicit Relu/Add nodes # ============================================================================ -spconv_fuse_implicit_gemm_relu = True +spconv_fuse_implicit_gemm_relu = False # Fuse SparseConv + BN in ``pts_middle_encoder`` before ONNX export (eval-mode Conv-BN fold). # Produces a BN-free sparse subgraph in the exported ONNX. -fuse_spconv_bn = True +fuse_spconv_bn = False # ============================================================================ # Checkpoint Path @@ -53,23 +53,25 @@ export = dict( mode="both", - work_dir="work_dirs/bevfusion_deployment_2_8", - onnx_path="work_dirs/bevfusion_deployment_2_8/onnx", + work_dir="work_dirs/bevfusion_deployment_2_8_no_opt", + onnx_path="work_dirs/bevfusion_deployment_2_8_no_opt/onnx", + # Dataset index of the sample used to trace/shape the exported model (read by ExportConfig, + # same as CenterPoint's export.sample_idx). + sample_idx=0, ) -# Optional: keep split component definitions for debugging, but export/eval as one main body. +# Optional: keep split component definitions but also emit one merged full-graph ONNX/engine. # - False: split sparse+dense ONNX/engine (default) # - True : one ONNX + one engine + one backend pipeline bevfusion_merge = dict( enabled=True, - onnx_file="bevfusion_lidar_fp16_opt.onnx", - engine_file="bevfusion_lidar_fp16_opt.engine", + onnx_file="bevfusion_lidar.onnx", + engine_file="bevfusion_lidar.engine", ) _WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" _TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" # ============================================================================ @@ -92,7 +94,9 @@ "voxels": {0: "voxels_num"}, "coors": {0: "voxels_num"}, "num_points_per_voxel": {0: "voxels_num"}, - "lidar_bev": {0: "batch", 2: "bev_h", 3: "bev_w"}, + # lidar_bev output is a fixed dense grid [1, C*D, 180, 180] regardless of voxel + # count; marking its batch/H/W dynamic re-dynamizes the boundary and defeats + # constant folding (adds ~50 shape-glue nodes). Keep it static. }, ), tensorrt_profile=dict( @@ -126,9 +130,10 @@ dict(name="label_pred", dtype="int64"), ], # Spatial dims must stay at head BEV resolution; see module docstring. - dynamic_axes={ - "lidar_bev": {0: "batch"}, - }, + # lidar_bev input is fully static ([1,256,180,180]; TRT profile is min=opt=max), + # so no dynamic axes — this lets constant folding remove the head's shape-glue + # (~177 nodes) and aligns the split graph node count with the monolithic export. + dynamic_axes={}, ), # H,W fixed to head grid (grid_size // out_size_factor). Widen only batch dim if needed. tensorrt_profile=dict( @@ -143,7 +148,6 @@ runtime_io = dict( info_file="info/t4dataset_j6gen2_base_infos_test.pkl", - sample_idx=0, ) onnx_config = dict( @@ -162,20 +166,16 @@ ) evaluation = dict( - enabled=True, + enabled=False, num_samples=5, num_warmup=2, verbose=True, + # ONNX *inference* is unsupported for BEVFusion. backends=dict( pytorch=dict( enabled=False, device=devices["cuda"], ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), tensorrt=dict( enabled=True, device=devices["cuda"], diff --git a/deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md b/deployment/projects/bevfusion_l/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md similarity index 92% rename from deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md rename to deployment/projects/bevfusion_l/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md index 17eec1fe3..7b84b5cc5 100644 --- a/deployment/projects/bevfusion/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md +++ b/deployment/projects/bevfusion_l/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md @@ -51,7 +51,7 @@ if coors.shape[1] == 3: 在新 framework 的 ONNX wrapper 中加入與舊版一致的正規化。 -- 檔案：`deployment/projects/bevfusion/export/onnx_export_pipeline.py` +- 檔案：`deployment/projects/bevfusion_l/export/onnx_export_pipeline.py` - 函式：`_normalize_sparse_coors_for_autoware()` - 行為：`[N,3] coors` 先 `flip(-1)`，再補 batch 欄位 @@ -61,8 +61,8 @@ if coors.shape[1] == 3: 在 framework inference backend（非 metric/evaluator）端加入同樣契約對齊。 -- `deployment/projects/bevfusion/pipelines/onnx.py` -- `deployment/projects/bevfusion/pipelines/tensorrt.py` +- `deployment/projects/bevfusion_l/pipelines/onnx.py` +- `deployment/projects/bevfusion_l/pipelines/tensorrt.py` - 函式：`_normalize_coors_for_legacy_main_body_contract()` - 行為：餵 backend 前，對 `[N,3]` coors 做 `flip(-1)` @@ -77,19 +77,19 @@ flip 必須放在模型邊界（export wrapper / backend preprocess），且只 該 commit（`chore: temp fix`）實際做了以下 5 項： -1. 新增 `deployment/projects/bevfusion/io/coors_contract.py` +1. 新增 `deployment/projects/bevfusion_l/io/coors_contract.py` - 統一定義兩個轉換函式： - `voxel_indices_xyz_to_graph_input_zyx()` - `graph_input_zyx_to_model_indices_xyz()` -2. 修改 `deployment/projects/bevfusion/export/onnx_export_pipeline.py` +2. 修改 `deployment/projects/bevfusion_l/export/onnx_export_pipeline.py` - `_normalize_sparse_coors_for_autoware()` 改為呼叫 `coors_contract`，而非散落的 `flip` - `_voxelize()` 明確把 voxel layer 輸出的 `[x,y,z]` 轉成 graph input `[z,y,x]` -3. 修改 `deployment/projects/bevfusion/pipelines/onnx.py` +3. 修改 `deployment/projects/bevfusion_l/pipelines/onnx.py` - backend 餵入前改用 `coors_contract` 做 `xyz -> zyx` -4. 修改 `deployment/projects/bevfusion/pipelines/tensorrt.py` +4. 修改 `deployment/projects/bevfusion_l/pipelines/tensorrt.py` - backend 餵入前改用 `coors_contract` 做 `xyz -> zyx` 5. 新增測試型 config - - `deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_on_board_test.py` + - `deployment/projects/bevfusion_l/config/deploy_config_split_fp16_opt_on_board_test.py` --- diff --git a/deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md b/deployment/projects/bevfusion_l/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md similarity index 94% rename from deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md rename to deployment/projects/bevfusion_l/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md index 9bceb4af1..93945d14c 100644 --- a/deployment/projects/bevfusion/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md +++ b/deployment/projects/bevfusion_l/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md @@ -50,19 +50,19 @@ split export 是兩次 trace： - dense wrapper：`BEVFusionDenseWrapper` - merge：`onnx.compose.merge_models` -```223:231:/home/yihsiangfang/ml_workspace/AWML/deployment/projects/bevfusion/export/onnx_export_pipeline.py +```223:231:/home/yihsiangfang/ml_workspace/AWML/deployment/projects/bevfusion_l/export/onnx_export_pipeline.py def _export_split(...): """Export ``bevfusion_sparse.onnx`` and ``bevfusion_dense.onnx``.""" ``` ### 2.1.1 分開 trace vs 一起 trace：各自用什麼方式 -- **一起 trace（single-file）** - - 入口：`BEVFusionONNXExportPipeline.export()` 的 single-file 路徑 - - 包裝：`BEVFusionMainBodyWrapper` - - 方法：呼叫一次 `self._export_to_onnx(...)`，直接把 +- **一起 trace（single-file）** — ⚠️ 此路徑已從 codebase 移除，僅保留於此作為 trace 差異的對照說明 + - (舊) 包裝：`BEVFusionMainBodyWrapper`（已刪除） + - (舊) 方法：呼叫一次匯出，直接把 `voxels/coors/num_points_per_voxel -> bbox_pred/score/label` 全流程 trace 成同一張 ONNX - 特性：shape 推導鏈通常完整保留在同一張圖內（較常看到 `Shape/Gather/Unsqueeze`） + - 現況：全圖 `bevfusion_merged` 改由 split 的 sparse+dense ONNX 事後 compose 而成（見下方 §2.2） - **分開 trace（split export）** - 入口：`BEVFusionONNXExportPipeline._export_split()` @@ -76,7 +76,7 @@ def _export_split(...): split merge 之後會做 `cleanup().toposort()`，一些跨段的中介 shape 節點會被折疊或改寫： -```488:490:/home/yihsiangfang/ml_workspace/AWML/deployment/projects/bevfusion/export/onnx_export_pipeline.py +```488:490:/home/yihsiangfang/ml_workspace/AWML/deployment/projects/bevfusion_l/export/onnx_export_pipeline.py merged_graph.cleanup().toposort() onnx.save_model(gs.export_onnx(merged_graph), str(merged_path)) ``` diff --git a/deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md b/deployment/projects/bevfusion_l/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md similarity index 95% rename from deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md rename to deployment/projects/bevfusion_l/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md index 500ef5ecd..6df4c4121 100644 --- a/deployment/projects/bevfusion/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md +++ b/deployment/projects/bevfusion_l/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md @@ -14,8 +14,8 @@ how to run a 2.8 LiDAR model through the split FP16 export → TensorRT → eval ```bash # inside the awml-bevfusion container, /workspace = host AWML -python -m deployment.cli.main bevfusion \ - deployment/projects/bevfusion/config/deploy_config_split_fp16_opt_2_8.py \ +python -m deployment.cli.main bevfusion_l \ + deployment/projects/bevfusion_l/config/deploy_config.py \ projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m_t4metric_v2.py ``` @@ -67,7 +67,7 @@ Dimension mismatch for tensor voxels and profile 0. At dimension axis 1, profile has min=10, opt=10, max=10 but tensor has 32. ``` -Fixed in `config/deploy_config_split_fp16_opt_2_8.py`: +Fixed in `config/deploy_config.py`: ```python voxels=dict(min_shape=[1, 32, 5], opt_shape=[64000, 32, 5], max_shape=[256000, 32, 5]) ``` @@ -78,7 +78,7 @@ This mirrors `projects/BEVFusion/deploy/utils.py` (`max_num_points 10 → 32`) a ### 3.2 ONNX opset 17 → 18 2.8 bumped `opset_version` to **18** in `projects/BEVFusion/configs/deploy/bevfusion_main_body_lidar_only*_tensorrt_dynamic.py`. -Applied to `config/deploy_config_split_fp16_opt_2_8.py` (`onnx_config.opset_version = 18`). +Applied to `config/deploy_config.py` (`onnx_config.opset_version = 18`). ### 3.3 What did NOT need porting - **`purge_mmdeploy_symbolics(["layer_norm"])`** (added to `projects/BEVFusion/deploy/exporter.py` @@ -120,7 +120,7 @@ into `projects/`, and renamed some config APIs. To run 2.8 these were also neede - `projects/BEVFusion/bevfusion/bevfusion.py` — added `_align_lidar_bev_to_head_grid` (the dense export wrapper calls it to assert BEV grid == head grid). **Only** this method was added; the rest of the 2.8 `bevfusion.py` is unchanged. -- `deployment/projects/bevfusion/export/onnx_export_pipeline.py` — +- `deployment/projects/bevfusion_l/export/onnx_export_pipeline.py` — `config.onnx_config` → `config.deploy_cfg.get("onnx_config", {})` (new framework keeps `_onnx_config` private; only `get_onnx_settings(component)` is public). - deploy config `runtime_io.info_file` → an existing `.pkl` diff --git a/deployment/projects/bevfusion_l/docs/29_README_ONNX_NODE_COUNT_ALIGNMENT.md b/deployment/projects/bevfusion_l/docs/29_README_ONNX_NODE_COUNT_ALIGNMENT.md new file mode 100644 index 000000000..8034baf2c --- /dev/null +++ b/deployment/projects/bevfusion_l/docs/29_README_ONNX_NODE_COUNT_ALIGNMENT.md @@ -0,0 +1,312 @@ +# 29: BEVFusion ONNX 節點數對齊 — commit `78b66a70` 的 clean-export 改動、split 邊界 `dynamic_axes`、以及新舊方法的殘留差異 + +本文件回答一個具體問題: + +> 用**新方法**(deployment CLI,split sparse+dense 再 merge)匯出的 `bevfusion_lidar.onnx`, +> 為什麼節點數跟**舊方法**(`projects/BEVFusion/deploy/torch2onnx.py`,整體單圖匯出)不一樣? +> 這跟 `78b66a70a1e6b394e74c1912c9c78d4258e7d6db`(BEVFusion 2.8.x release, #217)有關嗎?能不能對齊? + +結論先講: + +- **能對齊。** 關鍵是把 split config 裡 `lidar_bev` 的 `dynamic_axes` 拿掉,讓切點回到靜態 shape。 +- commit `78b66a70` **提供了** clean-export 的機制(把動態 `.shape`/`.dense()` 換成 config 來的靜態 shape), + 但這機制**必須配合「切點不標 dynamic」才會生效**;先前 split config 對 `lidar_bev` 標了 dynamic,把 commit 的靜態 shape 又動態化,效果被抵消。 +- 修正後仍有**個位數的殘留差異**,那是 split-then-merge 與 monolithic 在切點本質上的接縫差異,不是膨脹,且數值等價。 + +> ⚠️ 本文件**更正** [`26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](./26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) 的一項結論。 +> 詳見下方 [§7 更正 doc 26](#7-更正-doc-26)。 + +--- + +## 0. 實測數字(apples-to-apples) + +三份都是 **LiDAR-only、opset 18、`simplify=False`、無 fusion**,唯一差別是匯出方式與 `lidar_bev` 的 dynamic 設定: + +| 版本 | 匯出方式 | `lidar_bev` dynamic | #nodes | #initializers | 輸出 shape | +|---|---|---|---:|---:|---| +| **舊** `torch2onnx.py` | 整體單圖 | (無此邊界) | **423** | 207 | 符號 `Concat.../ReduceMax...` | +| **新** split(**修正前**) | sparse+dense→merge | `batch, H, W` 皆動態 | **524** | 207 | `[10, dyn]` | +| **新** split(**修正後**) | sparse+dense→merge | **全靜態** | **416** | 207 | **全靜態** `[10,500]` / `[500]` / `[500]` | + +- 修正前 split 比舊版多 **101** 個節點,幾乎全是 shape-plumbing(`Shape/Gather/Unsqueeze/Concat/Constant`)。 +- 修正後掉到 **416**,與舊版 423 幾乎一致(甚至少 7),initializer 三者都是 207。 +- 修正後輸出 shape **比舊版更乾淨**(全靜態),因為靜態邊界讓 shape inference 能一路推到底。 + +測試模型 / 指令見 [§8 重現步驟](#8-重現步驟)。 + +--- + +## 1. 背景:兩條匯出路線 + +BEVFusion LiDAR-only 有兩條 PyTorch→ONNX 路線,產生的圖**在數學上等價、但結構不同**: + +### 1.1 舊:整體單圖(monolithic) + +`projects/BEVFusion/deploy/torch2onnx.py --module main_body` + +- 一次 `torch.onnx.export`,把 `voxels/coors/num_points_per_voxel → (sparse encoder) → (dense backbone/neck/head) → bbox_pred/score/label` 全流程 trace 成**同一張** ONNX。 +- 後處理只有:onnx-graphsurgeon 把 TopK 的 `K` 改成常數([`exporter.py:_fix_onnx_graph`](../../../../projects/BEVFusion/deploy/exporter.py))+ `cleanup().toposort()`。 +- **沒有** onnx-simplifier。 +- `dynamic_axes` 只標了三個 sparse 輸入(`voxels/coors/num_points_per_voxel` 的第 0 維 `voxels_num`)。中間張量(如 sparse 輸出的 BEV feature)是圖內部,shape 在 trace 時是**具體數字**。 + +### 1.2 新:split sparse+dense 再 merge + +`python -m deployment.cli.main bevfusion_l ` + +- 把模型拆成兩段,各自 `torch.onnx.export`: + 1. `bevfusion_sparse.onnx`:`voxels/coors/num_points_per_voxel → lidar_bev`(含 spconv 自訂 op `GetIndicePairsImplicitGemm` / `ImplicitGemm`) + 2. `bevfusion_dense.onnx`:`lidar_bev → bbox_pred/score/label`(純標準 ONNX op,可上一般 TensorRT) +- 再用 [`onnx.compose.merge_models`](../export/transforms.py) 加上 `sparse/`、`dense/` 前綴合併成 `bevfusion_lidar.onnx`,最後 `cleanup().toposort()`。 +- 拆分的目的:sparse 段(spconv/plugin)與 dense 段(純 TRT)可以分開處理/量化。 + +**核心差別**:新路線在 `lidar_bev` 這個張量處**切了一刀**,製造出一個真正的圖邊界(sparse 的輸出、dense 的輸入)。這正是節點數差異的來源(見 §5)。 + +--- + +## 2. commit `78b66a70` 做了什麼(ONNX 相關) + +這個 commit 是 **BEVFusion 2.8.x release**,主題之一就是「**更乾淨的 ONNX 匯出**」。它系統性地把「在 runtime 讀 `.shape`」和「slice 賦值」這類**難匯出**的寫法,換成「用 config 來的靜態常數」和「乾淨的單一 op」。逐項說明: + +### 2.1 `sparse_to_dense` — 靜態化 sparse→dense 邊界(最關鍵) + +新增檔案 [`custom_sparse_conv_tensor.py`](../../../../projects/BEVFusion/bevfusion/custom_sparse_conv_tensor.py),docstring 明寫: + +> *"This customization is used to support cleaner ONNX export of sparse convolutions."* + +它取代了 spconv 的 `out.dense()`: + +**舊寫法**(`sparse_encoder.py`,commit 前): +```python +spatial_features = out.dense() # spconv scatter，動態 shape +N, C, H, W, D = spatial_features.shape # ← 讀 .shape（動態） +spatial_features = spatial_features.permute(0, 1, 4, 2, 3).contiguous() +spatial_features = spatial_features.view(N, C * D, H, W) # ← 用讀來的 N/C/H/W/D +``` + +**新寫法**(`sparse_encoder.py`,commit 後): +```python +spatial_features = sparse_to_dense(out, batch_size, self.dense_output_shapes, self.output_channels) +spatial_features = spatial_features.permute(0, 4, 3, 1, 2).contiguous() +spatial_features = spatial_features.view( + batch_size, + self.output_channels * self.dense_output_shapes[2], # ← config 常數 C*D + self.dense_output_shapes[0], # ← config 常數 H=180 + self.dense_output_shapes[1], # ← config 常數 W=180 +) +``` + +`sparse_to_dense` 內部用手算的 linear index 把 features scatter 進一個**靜態大小**的 `torch.zeros([batch*H*W*D, C])`,再 `view`。重點:reshape 的目標維度來自 **config 的 `dense_output_shapes`(靜態 int)**,不再讀 `.shape`。 + +同時 `BEVFusionSparseEncoder.__init__` 也改了 signature:移除 `aug_features_min_values/max_values/num_aug_features`,新增 `dense_output_shapes`。新舊兩個 model config **都**設了: +```python +# projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_120m.py +sparse_dense_output_shapes = [180, 180, 2] +# .../BEVFusion-L/bevfusion_lidar_voxel_..._120m.py +dense_output_shapes=_base_.sparse_dense_output_shapes, +``` + +### 2.2 `HardSimpleVoxelSinCosEncoder` — 把 sin-cos 編碼折成一個 FMA + +新增檔案 [`bevfusion_voxel_encoder.py`](../../../../projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py)。原本 sparse encoder 裡的特徵增強(sin-cos / Fourier 編碼)被搬出來,並且把 +`((x - min) / (max - min)) * pi * exponents` 這串 **sub → div → mul** 代數化簡成 +`scale * x + bias` 一個 `torch.addcmul`(FMA),可折成單一 op。ONNX 因此少掉一串 element-wise 節點。 + +### 2.3 head 重構 — 全面去除動態 `.shape` 與 slice 賦值 + +[`bevfusion_head.py`](../../../../projects/BEVFusion/bevfusion/bevfusion_head.py)(+116 行)把 `predict` 路徑整個改成 export-friendly: + +| 舊寫法(難匯出) | 新寫法(乾淨) | +|---|---| +| `batch_size = inputs.shape[0]`;`fusion_feat.view(batch_size, C, -1)` | `fusion_feat.view(-1, self.share_conv_out_channels, self.spatial_dim)`(靜態 `spatial_dim = H*W`) | +| `bev_pos.repeat(batch_size,...).to(device)` | `bev_pos` 註冊成 buffer(靜態、自動上 GPU) | +| `local_max[:, idx, pad:-pad, pad:-pad] = ...`(**slice 賦值** → ONNX 變 ScatterND 惡夢) | `F.pad(...)` + `torch.cat(...)` + 用 `local_concat_class_remapping` buffer 做 gather 重排 | +| `heatmap.view(batch_size,...).argsort(...)[:num_proposals]` | `torch.topk(k=num_proposals)`(單一 TopK op) | +| `top_proposals // heatmap.shape[-1]` | `top_proposals // self.spatial_dim`(靜態) | + +### 2.4 `bevfusion.py` — onnx 路徑固定 `batch_size = 1` + +[`bevfusion.py`](../../../../projects/BEVFusion/bevfusion/bevfusion.py) 的 `extract_pts_feat` onnx 分支直接寫死 `batch_size = 1`(靜態 Python int),並把 voxel 特徵編碼交給新的 `pts_voxel_encoder`。`torch.cuda.amp.autocast` 也更新成 `torch.amp.autocast("cuda", ...)`。 + +### 2.5 `exporter.py` — LayerNorm 匯出成原生 `LayerNormalization` + +舊 deploy 路徑的 [`exporter.py`](../../../../projects/BEVFusion/deploy/exporter.py) 新增 `purge_mmdeploy_symbolics(["layer_norm"])`:刪掉 mmdeploy 對 `layer_norm` 的 symbolic,讓它匯出成 opset 17+ 的**原生 `LayerNormalization`**(而非被 mmdeploy 拆解)。這就是為什麼新舊兩張圖都看到 `LayerNormalization: 3`。 + +### 2.6 `utils.py` — `TransFusionBBoxCoder` 支援 per-class 分數門檻 + +[`utils.py`](../../../../projects/BEVFusion/bevfusion/utils.py) 的 `decode` 重構了 filter 邏輯,`score_threshold` 可接受 list/tuple(每類不同門檻,`score_threshold[final_preds]`)。與節點數無直接關係,但屬同一 release 的後處理改動。 + +--- + +## 3. 這些改動對 ONNX 的整體效果 + +一句話:**把「要到 runtime 才知道的形狀」變成「trace/編譯期就是常數的形狀」,讓 `do_constant_folding` 能把 shape-plumbing 折掉。** + +- `torch.onnx.export` 遇到 `x.view(...)` / `x.shape[i]` 這類操作,會產生一串 `Shape → Gather → Unsqueeze → Concat → Reshape` 在**執行時**算形狀。 +- `do_constant_folding=True`(新舊都有開)只在**這串的輸入全是常數**時,才能把它折成一個 Constant 並刪掉。 +- commit `78b66a70` 的每一項改動(`dense_output_shapes`、`spatial_dim`、`batch_size=1`、`F.pad`/`cat`/`topk`)都是在**把那些輸入變成常數**,於是整串 shape-plumbing 可以被折掉 → 圖變乾淨。 + +**但有一個前提**:這些靜態 shape 只有在「該維度沒有被宣告成 dynamic axis」時才留得住。一旦你在匯出時把某個維度列進 `dynamic_axes`,exporter 就**被迫**保留該維度的 shape-plumbing —— 這正是新方法先前的問題。 + +--- + +## 4. 為什麼「切一刀」會多出節點(機制) + +> 一個張量只要還在圖**內部**,shape 在 trace 時是具體數字 → 相關 shape 運算被折掉; +> 一旦在那裡**切成邊界**(輸出/輸入),shape 變成符號 → 折不掉,留成一堆 `Shape/Gather/Unsqueeze/Concat`。 + +以切點 `lidar_bev`(shape ≈ `[1, 512, 180, 180]`)為例: + +- **整體單圖(舊)**:`lidar_bev` 是圖內部 activation,trace 時 shape 具體 → dense head 的 shape 運算全折疊 → 乾淨。 +- **split(新)**:`lidar_bev` 變成 sparse 的 output + dense 的 input。**生產端**(sparse)要動態組出輸出形狀,**消費端**(dense)的 input 形狀是 placeholder → 兩側都得補一套 shape 重建。切一刀 → 兩邊各多一套,節點淨增。 + +實測子圖 shape-glue(`Shape+Gather+Unsqueeze+Concat+Constant+Cast`)節點:sparse ≈ 50、dense ≈ 177 —— 對應 §0 修正前多出的 ~101 節點。 + +--- + +## 5. 真正的病灶:split config 對 `lidar_bev` 標了 `dynamic_axes` + +即使 §2.1 的 `sparse_to_dense` 已經用靜態 shape 建好 `lidar_bev`,先前 split config 又把它標成 dynamic,把靜態成果**動態化**回去: + +`deployment/projects/bevfusion_l/config/deploy_config.py`(修正前): +```python +# bevfusion_sparse — 輸出 +dynamic_axes={ + "voxels": {0: "voxels_num"}, + "coors": {0: "voxels_num"}, + "num_points_per_voxel": {0: "voxels_num"}, + "lidar_bev": {0: "batch", 2: "bev_h", 3: "bev_w"}, # ← 把 batch/H/W 標成動態 +}, +# bevfusion_dense — 輸入 +dynamic_axes={ + "lidar_bev": {0: "batch"}, # ← 把 batch 標成動態 +}, +``` + +一旦 `lidar_bev` 的 batch/H/W 被列進 `dynamic_axes`: + +1. **sparse 側**:輸出 shape 變 `['batch','Reshape...','bev_h','bev_w']`(全符號),`sparse_to_dense` 的靜態 view 被迫展開成 `Shape/Gather/Concat/Reshape`。 +2. **dense 側**:input batch 動態 → 整個 head 凡是依賴 batch 維的運算(reshape、gather、decode)都折不掉,留下大量 shape-plumbing。 + +**佐證:這些 dynamic axes 根本沒必要。** dense 的 TRT profile 是 +```python +lidar_bev=dict(min_shape=[1,256,180,180], opt_shape=[1,256,180,180], max_shape=[1,256,180,180]) +``` +`min == opt == max`,batch/H/W 執行期全鎖死;而且 config docstring 自己就寫「H/W 必須固定 180,不能給範圍,否則 `bbox_head` 的 `Reshape/Gather` 會壞掉、mAP 變垃圾」。既然執行期都是定值,標成 dynamic 純屬有害無益。 + +### 5.1 修正 + +把兩處 `lidar_bev` 移出 `dynamic_axes`,**保留** `voxels/coors/num_points_per_voxel` 的動態(voxel 數每幀真的會變,且只影響 sparse 前端、不會膨脹 head): + +```python +# bevfusion_sparse — 輸出：拿掉 lidar_bev（保留三個 sparse 輸入的動態） +dynamic_axes={ + "voxels": {0: "voxels_num"}, + "coors": {0: "voxels_num"}, + "num_points_per_voxel": {0: "voxels_num"}, +} +# bevfusion_dense — 輸入：完全靜態 +dynamic_axes={} +``` + +已同步套用到: +- [`config/deploy_config.py`](../config/deploy_config.py)(正式,fusion 開) +- [`config/deploy_config_without_opt.py`](../config/deploy_config_without_opt.py)(無 fusion / 無 simplify,用於 §0 的 apples-to-apples 比較) + +效果:節點 524 → **416**(見 §0)。 + +--- + +## 6. 修正後仍存在的差異(良性) + +對齊後 `416 vs 423`,op 層級只剩個位數差異(新-修正後 相對 舊): + +``` +Constant -6 Slice -2 Mul -1 Concat -1 Sub -1 ← 新版更精簡 +Reshape +2 Mod +1 Transpose +1 ← sparse→dense 接縫 +``` + +這些是 **monolithic vs split-then-merge 本質上的接縫差異**,不是 shape-glue 膨脹: + +1. **接縫重建**:`sparse_to_dense` 的 scatter/linear-index 計算(`Mod`、`Reshape`、`Transpose`)在 split 版被具體化在切點附近;monolithic 版因整圖 trace 而被折進常數路徑。 +2. **有幾項新版反而更少**(`Constant/Slice/Mul/Concat/Sub`),因為靜態邊界讓 folding 更徹底。 +3. 這種個位數差異已無法再靠改設定消除,除非放棄 split。 + +其他非 op-count 的差異: + +| 面向 | 舊(monolithic) | 新(split-merge) | +|---|---|---| +| producer | `pytorch 2.8.0` | `onnx.compose.merge_models 1.0` | +| opset_import | `ai.onnx 18` + `autoware 1` 各一次 | 兩組(每個子圖一組,merge 後保留;功能相同) | +| 節點名稱 | 無前綴 | `sparse/`、`dense/` 前綴 | +| 輸出 shape | 符號(`Concatbbox_pred_dim_0`...) | 全靜態(`[10,500]`...) | + +### 6.1 正式 config 額外的 fusion(與本文的邊界修正正交) + +[`deploy_config.py`](../config/deploy_config.py) 比 `deploy_config_without_opt.py` 多開兩個 fusion,會**再**減少節點,但這與 commit / 邊界修正無關,是獨立的優化旋鈕: + +- `fuse_spconv_bn = True`:把 sparse 段的 SparseConv+BN 在匯出前 fold(eval-mode Conv-BN 融合)→ 減少 `BatchNormalization`。 +- `spconv_fuse_implicit_gemm_relu = True`:把 `ImplicitGemm` 後的 `Relu`(及 `Add(const)+Relu`)烘進 plugin 的 `act_type` → 減少 `Relu`。 + +> 注意:兩個 config 的 `onnx_config.simplify` **都是 `False`**。本 repo 目前預設**不跑 onnx-simplifier**;§0 的比較也都是 `simplify=False`。若另外開 `simplify=True`,節點會再大幅下降(constant fold 進 initializer、shape 定死),但那是另一層優化,不影響本文結論。 + +--- + +## 7. 更正 doc 26 + +[`26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](./26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) §2–§3 主張「split export 的 shape 鏈通常**更短/更少**」。**依本文實測,在目前的 config 下結論相反**:split(524)比 monolithic(423)**更多**節點。原因: + +1. doc 26 沒有把 config 對 `lidar_bev` 宣告的 `dynamic_axes` 納入考量。一旦切點被標 dynamic,dense head 的 shape-plumbing **無法折疊**,反而**增加**節點(§5)。 +2. doc 26 引用的程式路徑(`out_tensor.dense()`、`_conv_out_to_bev`、`BEVFusionDenseWrapper`、`onnx_export_pipeline.py:_export_split`)是 **refactor 前的舊碼**,現已不存在;現行實作是 `sparse_to_dense`([`custom_sparse_conv_tensor.py`](../../../../projects/BEVFusion/bevfusion/custom_sparse_conv_tensor.py))+ [`export/transforms.py`](../export/transforms.py)+ [`export/component_builder.py`](../export/component_builder.py)。 + +doc 26 仍然正確的部分:**「節點長相/數量不同 ≠ 數值不同;應以契約一致性 + 數值驗證為準」**(§4–§6)。這點本文完全同意,見 §9。 + +--- + +## 8. 重現步驟 + +於 docker container `awml-bevfusion`(repo bind-mount 在 `/workspace`,torch 2.8.0 / onnx 1.17)內: + +```bash +# 新方法(split，會產出 sparse/dense/merged 三個 onnx) +python -m deployment.cli.main bevfusion_l \ + deployment/projects/bevfusion_l/config/deploy_config_without_opt.py \ + projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m_t4metric_v2.py +# → work_dirs/bevfusion_deployment_2_8_no_opt/onnx/bevfusion_lidar.onnx +# 只想匯 ONNX、跳過 TRT 建置：暫時把 export.mode 設成 "onnx" + +# 舊方法(monolithic) +python projects/BEVFusion/deploy/torch2onnx.py \ + projects/BEVFusion/configs/deploy/bevfusion_main_body_lidar_only_tensorrt_dynamic.py \ + projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m.py \ + work_dirs/bevfusion/bevfusion_2_8/best_epoch_25.pth --device cuda:0 \ + --work-dir work_dirs/bevfusion/bevfusion_2_8/ --module main_body +# → work_dirs/bevfusion/bevfusion_2_8/bevfusion_lidar.onnx +``` + +比對節點數 / op 分佈(host 或 container 皆可,只需 `onnx`): + +```python +import onnx +from collections import Counter +for p in [OLD_PATH, NEW_PATH]: + g = onnx.load(p).graph + print(p, "#nodes", len(g.node), "#init", len(g.initializer)) + print(sorted(Counter(n.op_type for n in g.node).items())) +``` + +--- + +## 9. 結論與建議 + +- **節點數對齊 = 已完成**:移除 `lidar_bev` 的 `dynamic_axes` 後,split 版 416 ≈ monolithic 版 423,差異只剩良性接縫。 +- **與 commit `78b66a70` 的關聯**:相關,但不是「新方法沒 apply」。commit 的 `sparse_to_dense`/靜態 head shape **有 apply**;先前是被 split config 的 `dynamic_axes` 抵消。修正後,commit 的 clean-export 才真正落地(輸出甚至比舊版更靜態)。 +- **節點數不是等價判準**:如 doc 26 所述,不同 ONNX 表達可語義等價。真正該驗證的是**契約一致性**(`coors` 欄位序、BN/bias、head grid 尺寸)與**數值**(同一 sample 跑 TensorRT 比對 `lidar_bev` / `bbox_pred`/`score`/`label`)。節點數對齊只是讓兩條路線**更好比對、更好維護**,不是正確性的證明。 + +--- + +### 附:相關檔案 + +- 匯出流程:[`export/onnx_export_pipeline.py`](../export/onnx_export_pipeline.py)、[`export/component_builder.py`](../export/component_builder.py)、[`export/transforms.py`](../export/transforms.py)(`merge_split_sparse_dense_onnx`、TopK fix) +- config:[`config/deploy_config.py`](../config/deploy_config.py)、[`config/deploy_config_without_opt.py`](../config/deploy_config_without_opt.py) +- 模型:[`custom_sparse_conv_tensor.py`](../../../../projects/BEVFusion/bevfusion/custom_sparse_conv_tensor.py)、[`sparse_encoder.py`](../../../../projects/BEVFusion/bevfusion/sparse_encoder.py)、[`bevfusion_voxel_encoder.py`](../../../../projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py)、[`bevfusion_head.py`](../../../../projects/BEVFusion/bevfusion/bevfusion_head.py) +- 舊路徑:[`projects/BEVFusion/deploy/exporter.py`](../../../../projects/BEVFusion/deploy/exporter.py)、[`torch2onnx.py`](../../../../projects/BEVFusion/deploy/torch2onnx.py) diff --git a/deployment/projects/bevfusion_l/docs/30_README_EVALUATION_PIPELINE_WALKTHROUGH.md b/deployment/projects/bevfusion_l/docs/30_README_EVALUATION_PIPELINE_WALKTHROUGH.md new file mode 100644 index 000000000..84b36cedd --- /dev/null +++ b/deployment/projects/bevfusion_l/docs/30_README_EVALUATION_PIPELINE_WALKTHROUGH.md @@ -0,0 +1,588 @@ +# BEVFusion Deployment Evaluation — 完整逐步導覽 + +> 這份文件的目標:**把「BEVFusion 在 deployment pipeline 裡是怎麼被完整 evaluate 的」一步一步、 +> 一個 function 一個 function、一個檔案一個檔案地講清楚**,並且順帶解釋: +> +> 1. BEVFusion 模型裡的每一個部件(voxel layer / voxel encoder / sparse middle encoder / +> backbone / neck / bbox_head / bbox_coder)各自在做什麼; +> 2. ONNX 這邊的每一個部件(export pipeline / wrappers / components / 各種 graph transform / +> merge)各自在做什麼,以及跟 PyTorch 模型如何對應。 +> +> 內容全部以「實際原始碼」為準(依 `CLAUDE.md` 的規定:Graphify 當地圖、原始碼與測試是最終真相)。 +> 本文件所描述的檔案與行為在撰寫時已逐一讀過。父層架構地圖見 [`../README.md`](deployment/projects/bevfusion_l/README.md)。 + +--- + +## 0. 一句話總結 + +```text +一次 `python -m deployment.cli.main bevfusion_l ` 會跑 +Load checkpoint → Export → Verify → Evaluate 四個階段。 +Evaluate 階段對「每個啟用的 backend(PyTorch / TensorRT)」各跑一次: + 逐個 sample → 前處理(體素化)→ 跑模型(sparse+dense)→ 後處理(bbox 解碼) + → 把預測與 GT 丟進 T4MetricV2 相同的度量引擎 → 算出 mAP / mAPH / 延遲。 +``` + +Evaluation 本身完全 **backend 無關**:同一個 evaluation loop、同一套 metric,PyTorch 與 TensorRT +只是換了 pipeline 實作;PyTorch 是「reference / 參考真值」,TensorRT 是「部署後要驗收的對象」。 + +--- + +## 1. 角色分工:誰負責什麼 + +deployment 框架把一次部署切成「階段(stage)」,每個 top-level 目錄是一個階段;`projects/bevfusion_l/` +則是實作同名階段的「專案 bundle」。與 **evaluation** 有關的角色: + +| 層 | 檔案 | 職責(evaluation 視角) | +| --- | --- | --- | +| CLI | [`deployment/cli/main.py`](deployment/cli/main.py) | 發現/註冊專案、建 argparser、把 `args` 交給 adapter | +| CLI | [`deployment/cli/args.py`](deployment/cli/args.py) | `deploy_cfg` / `model_cfg` / `--log-level` 參數 + logging | +| 註冊表 | [`deployment/projects/registry.py`](deployment/projects/registry.py) | `bevfusion` 名字 → `run()` 的對照 | +| 入口 | [`deployment/projects/bevfusion_l/entrypoint.py`](deployment/projects/bevfusion_l/entrypoint.py) | **組裝**:config + data_loader + executor + evaluator + runner | +| Runner | [`deployment/projects/bevfusion_l/runner.py`](deployment/projects/bevfusion_l/runner.py) | 載入 PyTorch 模型;把 export/verify/eval 串起來 | +| Runner(共用) | [`deployment/runtime/runner.py`](deployment/runtime/runner.py) | `BaseDeploymentRunner.run()`:Export→Verify→Evaluate | +| 協調器 | [`deployment/runtime/evaluation_orchestrator.py`](deployment/runtime/evaluation_orchestrator.py) | 決定「要對哪些 backend、在哪個 device」跑 eval | +| 協調器 | [`deployment/runtime/export_orchestrator.py`](deployment/runtime/export_orchestrator.py) | 載入 checkpoint、(可選)匯出 ONNX/TRT、解析 artifact 路徑 | +| 資料 | [`deployment/io/point_cloud_data_loader.py`](deployment/io/point_cloud_data_loader.py) | 用 MMDet3D test pipeline 產出 `points` / `metainfo` / `ground_truth` | +| 執行原語 | [`deployment/execution/backend_executor.py`](deployment/execution/backend_executor.py) | 「建 pipeline + 準備 input + 管 device」的抽象 | +| 執行原語 | [`deployment/projects/bevfusion_l/evaluation/executor.py`](deployment/projects/bevfusion_l/evaluation/executor.py) | BEVFusion 版:依 backend 建 PyTorch/TRT pipeline | +| Evaluator | [`deployment/evaluation/base_evaluator.py`](deployment/evaluation/base_evaluator.py) | **核心 evaluation loop**(warmup→逐 sample→累積) | +| Evaluator | [`deployment/evaluation/detection_3d_evaluator.py`](deployment/evaluation/detection_3d_evaluator.py) | 3D 偵測的 pred/GT 解析、結果彙整、列印 | +| Pipeline | [`deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py) | 前處理/後處理 + sparse/dense 兩段接縫(base) | +| Pipeline | [`.../inference/pytorch_inference_pipeline.py`](deployment/projects/bevfusion_l/inference/pytorch_inference_pipeline.py) | PyTorch backend 的 sparse/dense 實作 | +| Pipeline | [`.../inference/tensorrt_inference_pipeline.py`](deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py) | TensorRT backend(split 雙引擎 / merged 單引擎) | +| Metrics | [`deployment/metrics/detection_3d_metrics.py`](deployment/metrics/detection_3d_metrics.py) | 把 pred/GT 轉 perception_eval 物件、算 mAP/mAPH | +| Metrics(共用) | [`deployment/metrics/base_metrics_interface.py`](deployment/metrics/base_metrics_interface.py) | frame buffer、evaluator 生命週期、快取 | +| Metrics(共用) | [`deployment/metrics/detection_base.py`](deployment/metrics/detection_base.py) | `MetricsScore` → 扁平 dict / `DetectionSummary` | +| Config | [`.../config/bevfusion_deployment_config.py`](deployment/projects/bevfusion_l/config/bevfusion_deployment_config.py) | BEVFusion 專屬旗標 + 解析最終 component layout | +| Config | [`.../config/component_layout.py`](deployment/projects/bevfusion_l/config/component_layout.py) | split / merged 判斷、衍生 `bevfusion_merged` | + +--- + +## 2. 一次完整 evaluation 的呼叫鏈(bird's-eye) + +```mermaid +flowchart TD + A["cli/main.py: main(argv)"] --> B["registry: adapter.run(args)"] + B --> C["bevfusion/entrypoint.py: run(args)"] + C --> C1["建 BEVFusionDeploymentConfig"] + C --> C2["建 PointCloudDataLoader（MMDet3D 資料集）"] + C --> C3["extract_t4metric_v2_config（度量設定）"] + C --> C4["建 BEVFusionExecutor（backend 原語）"] + C --> C5["建 Detection3DEvaluator（+ Detection3DMetricsInterface）"] + C --> C6["建 BEVFusionDeploymentRunner"] + C6 --> D["runtime/runner.py: run()"] + D --> E["export_orchestrator.run()：載入 PyTorch 模型"] + E --> F["executor.set_pytorch_model(model)"] + F --> G["verification_orchestrator.run()（本 config 關閉）"] + G --> H["evaluation_orchestrator.run()"] + H --> H1["_resolve_model_specs()：哪些 backend 有 artifact"] + H1 --> I["對每個 backend：evaluator.evaluate(model_spec, ...)"] + I --> J["base_evaluator.evaluate()：warmup + 逐 sample loop"] + J --> J1["executor.create_pipeline() → PyTorch / TRT pipeline"] + J --> J2["executor.prepare_input(sample) → points + metainfo"] + J --> K["pipeline.infer()：preprocess→run_model→postprocess"] + K --> L["_parse_predictions / _parse_ground_truths"] + L --> M["metrics_interface.add_frame()（buffer）"] + M --> N["_build_results()：compute_metrics() → mAP/mAPH/latency"] + N --> O["print_results() + 跨 backend 比較表"] +``` + +以下把每一步展開。 + +--- + +## 3. Stage 0 — CLI 與組裝 + +### 3.1 `cli/main.py`(進入點) +- [`main(argv)`](deployment/cli/main.py#L84):建 parser → `parser.parse_args` → `project_registry.get(args._adapter_name)` → `adapter.run(args)`。 +- [`build_parser()`](deployment/cli/main.py#L40): + - [`_discover_project_packages()`](deployment/cli/main.py#L22) 掃描 `deployment/projects/` 下的子套件名(不 import)。 + - [`_import_and_register_project()`](deployment/cli/main.py#L35) `import deployment.projects.bevfusion`,觸發該套件的 **副作用註冊**。 + - 每個成功註冊的專案配一個 subparser,並 `parse_base_args(sub)` 加上 `deploy_cfg` / `model_cfg` / `--log-level`。 + +### 3.2 註冊是怎麼發生的 +- [`deployment/projects/bevfusion_l/__init__.py`](deployment/projects/bevfusion_l/__init__.py) 在 import 時就 + `project_registry.register(ProjectAdapter(name="bevfusion", run=run))`,其中 `run` 就是 entrypoint 的 `run`。 +- [`ProjectRegistry`](deployment/projects/registry.py#L35) 只是 `name → run` 的字典;沒有任何 per-project CLI flag—— + **所有會影響產物的東西都寫在 deploy config**,確保可版本控管、可重現。 + +### 3.3 `entrypoint.py: run(args)` — 把所有零件組起來 +[`run()`](deployment/projects/bevfusion_l/entrypoint.py#L18) 依序: + +1. `setup_logging` 設 logger。 +2. `Config.fromfile(args.deploy_cfg)` 與 `Config.fromfile(args.model_cfg)` 讀兩份 MMEngine config。 +3. **`config = BEVFusionDeploymentConfig(deploy_cfg)`**:建構時就把最終 component layout 解析定案(§9)。 +4. (可選)`add_deployment_file_logging` 把 log 也寫檔。 +5. `PointCloudDataLoader(info_file, model_cfg)`:建資料集(§5)。 +6. **`metrics_config = extract_t4metric_v2_config(model_cfg)`**:從 `model_cfg.val_evaluator` 抽出跟訓練期 + T4MetricV2 完全相同的度量設定(§8.1)。 +7. `plugin_libraries`:從 `deploy_cfg.tensorrt_config.plugin_libraries` 取 spconv ImplicitGemm plugin 的 `.so` 路徑。 +8. **`executor = BEVFusionExecutor(components_cfg, plugin_libraries)`**:一個 executor 實例,evaluator 與 runner 共用。 +9. **`evaluator = Detection3DEvaluator(model_cfg, metrics_config, executor)`**。 +10. **`runner = BEVFusionDeploymentRunner(data_loader, evaluator, executor, config, model_cfg)`**。 +11. `runner.run()`。 + +> 關鍵:`executor`、`evaluator`、`data_loader` 都是在這裡「一次建好、之後共用」。executor 這時還沒有 +> PyTorch 模型(`pytorch_model=None`);模型要等 export 階段載入後,由 runner 回填(§4.2)。 + +--- + +## 4. Stage 1–4 — Runner 把四階段串起來 + +### 4.1 `BEVFusionDeploymentRunner.__init__` +[`runner.py`](deployment/projects/bevfusion_l/runner.py#L51):在呼叫 `super().__init__` **之前** 先建好 ONNX pipeline +(因為 base runner 會把它直接轉交給 ExportOrchestrator,沒有事後注入的位置): + +```python +onnx_pipeline = OnnxExportPipeline( + sample_extractor=BEVFusionSampleExtractor(), # §10.1 + component_builder=BEVFusionComponentBuilder(config), # §10.2 + finalize=bevfusion_merge_finalize if config.merge_bevfusion else None, # §10.6 +) +``` + +然後 `super().__init__()` 進到共用的 [`BaseDeploymentRunner.__init__`](deployment/runtime/runner.py#L51),它建立四個協調器: +- `ArtifactManager(config)` +- `ExportOrchestrator(...)`(拿到上面的 onnx_pipeline + 預設 `TensorRTExportPipeline`) +- `VerificationOrchestrator(config, verifier, data_loader, artifact_manager)` +- **`EvaluationOrchestrator(config, evaluator, data_loader, artifact_manager)`** ← 本文重點 + +### 4.2 `BaseDeploymentRunner.run()`(evaluation 的觸發點) +[`run()`](deployment/runtime/runner.py#L99) 只有五步: + +```python +export_result = self.export_orchestrator.run() # 1) 載入 PyTorch 模型（見下） +self._executor.set_pytorch_model(export_result.pytorch_model) # 2) 回填模型給共用 executor +results.verification_results = self.verification_orchestrator.run() # 3) 交叉驗證（本 config 關閉） +results.evaluation_results = self.evaluation_orchestrator.run() # 4) ★ evaluation +``` + +**第 1 步為什麼要「export」才能 evaluate?** 因為即使 `export.mode="none"`(本 deploy_config 就是), +[`ExportOrchestrator.run()`](deployment/runtime/export_orchestrator.py#L84) 仍會 **一定** 先 +[`_load_and_register_pytorch_model()`](deployment/runtime/export_orchestrator.py#L136),它呼叫 runner 覆寫的 +[`load_pytorch_model()`](deployment/projects/bevfusion_l/runner.py#L83) → +[`build_bevfusion_model()`](deployment/projects/bevfusion_l/io/model_loader.py#L21): +- `build_mmdet3d_model(model_cfg, checkpoint, cuda)` 建圖並 `load_checkpoint`,設 eval 模式; +- `_require_lidar_only_bevfusion(model)` 確認是 LiDAR-only(無 `fusion_layer`/`img_backbone`、有 `pts_middle_encoder`); +- 若 `fuse_spconv_bn=True`,對 `pts_middle_encoder` 做 SparseConv+BN 折疊(§10.4)。 + +runner 的 `load_pytorch_model` 也在這裡 `set_do_sort(config.spconv_do_sort)`——`do_sort` 是被 spconv 於 ONNX symbolic 與 forward 讀取的 process 全域,只需在匯出/推論前設一次,放在 runner(而非 component builder)是因為它是 deploy 全域設定、不是 per-component 的事。 + +這顆模型有三個用途:(a) ONNX 匯出的來源;(b) **PyTorch backend 的推論本體**;(c) 所有 backend 後處理都要用它的 +`bbox_head.bbox_coder` 解碼、用它的 `pts_voxel_layer` 體素化(§6、§7)。所以 `set_pytorch_model` 之後, +PyTorch / TensorRT 兩個 backend 的前後處理共用同一顆參考模型。 + +> 在 `deploy_config.py` 裡 `export.mode="none"`,所以這一輪 **不會** 重新產生 ONNX / engine。 +> TensorRT 評估用的 engine 由 `evaluation.backends.tensorrt.engine_dir` 指到既有的 `work_dir/tensorrt/`。 +> 若要重新匯出,把 `export.mode` 設成 `onnx` / `trt` / `both`(§10 詳述匯出流程)。 + +--- + +## 5. 資料從哪來 — `PointCloudDataLoader` + +[`point_cloud_data_loader.py`](deployment/io/point_cloud_data_loader.py):CenterPoint 與 BEVFusion 共用同一個 +point-cloud loader(BEVFusion 目前以 **LiDAR-only** 部署)。 + +- `__init__` → [`_build_dataset()`](deployment/io/point_cloud_data_loader.py#L52): + `init_default_scope("mmdet3d")`,深拷貝 `model_cfg.test_dataloader.dataset`,若 deploy config 有 `runtime_io.info_file` + 就覆寫 `ann_file`,設 `test_mode=True`,再 `DATASETS.build(...)`。這代表 **evaluation 用的資料與前處理完全等同訓練時的 test pipeline**。 +- [`load_sample(index)`](deployment/io/point_cloud_data_loader.py#L64):取 `dataset[index]`,回傳一個 `SampleData` TypedDict: + - `points`:`[N, point_features]` 的點雲張量(CPU); + - `metainfo`:給後處理用的樣本 metadata; + - `ground_truth`:來自 `data_samples.eval_ann_info`(含 `gt_bboxes_3d` / `gt_labels_3d` / `num_lidar_pts`)。 +- [`num_samples`](deployment/io/point_cloud_data_loader.py#L90):資料集長度;evaluation 若設 `num_samples=-1` 就評估全部。 + +> `load_sample` 一定要帶 `ground_truth`,否則 evaluation loop 會丟 `KeyError`(見 §7.2)。 + +--- + +## 6. BEVFusion 模型的每一個部件在做什麼 + +在講 evaluation loop 之前,先把「模型本體」拆開,因為 sparse/dense 兩段接縫、ONNX 的切法,全都對應到這些部件。 +LiDAR-only BEVFusion 的資料流: + +```text +points ──(pts_voxel_layer 體素化)──▶ voxels / coors / num_points_per_voxel + └── 這一步在 ONNX 圖「之外」,由前處理做 + +voxels/coors/num_points + │ + ├─ pts_voxel_encoder ── 每個 voxel 內做 mean-pool + sin/cos Fourier 特徵 ─▶ voxel_features + │ + └─ pts_middle_encoder (spconv 稀疏卷積塔) ── 把稀疏 voxel 散佈成稠密 BEV ─▶ lidar_bev [B, C, H, W] + └── 這兩步合稱「sparse 分支」，對應 ONNX 的 bevfusion_sparse + +lidar_bev [B,256,H,W] + │ + ├─ pts_backbone (SECOND) ─────────▶ 多尺度特徵 + ├─ pts_neck (SECONDFPN) ──────────▶ 融合後 BEV + ├─ _align_lidar_bev_to_head_grid ─▶ 對齊到 head 的 grid（grid_size // out_size_factor，如 180×180） + └─ bbox_head (transformer 偵測頭) ─▶ head 輸出 dict（heatmap / query_labels / center / height / dim / rot / vel ...） + └── 這四步合稱「dense 分支」，對應 ONNX 的 bevfusion_dense +``` + +各部件職責: + +| 部件 | 屬性名 | 做什麼 | +| --- | --- | --- | +| 體素化 | `pts_voxel_layer` | 把點雲切成硬體素(hard voxelization),輸出 `(voxels, coors, num_points_per_voxel)`。**在 ONNX 圖外**,前處理階段呼叫。 | +| Voxel encoder | `pts_voxel_encoder` | 對每個 voxel 內的點做 mean-pool,再加位置的 sin/cos Fourier 特徵,得到每 voxel 的特徵向量。 | +| Sparse middle encoder | `pts_middle_encoder` | spconv 稀疏卷積塔;把稀疏 voxel 特徵散佈/卷積成稠密 BEV 特徵圖 `lidar_bev`。**這是需要 Autoware ImplicitGemm plugin 的部分**。 | +| Backbone | `pts_backbone` | SECOND:2D 卷積 backbone,抽多尺度 BEV 特徵。 | +| Neck | `pts_neck` | SECONDFPN:上採樣/融合多尺度特徵。 | +| Grid 對齊 | `_align_lidar_bev_to_head_grid` | 把 SECOND/FPN 的 BEV 解析度池化到 head 期望的 grid(否則 transformer decoder 的 `key` 與 `key_pos` 長度不合)。 | +| 偵測頭 | `bbox_head` | Transformer 偵測頭;輸出 `heatmap` / `query_labels` / `query_heatmap_score` 及 `center/height/dim/rot/vel` 迴歸,並在圖內做 query 選取(含 TopK)。 | +| 解碼器 | `bbox_head.bbox_coder` | 把 head 的編碼輸出解回公制座標的 3D box(後處理用,見 §7.4)。 | + +**head 輸出 → 三個張量的契約**: +[`head_dict_to_detection_outputs()`](deployment/projects/bevfusion_l/io/head_outputs.py#L15) 是 **唯一** 把 head dict +轉成 `(bbox_pred, score, label)` 的地方,PyTorch 與 ONNX 兩邊都呼叫它,確保輸出契約位元級一致: +- `score = sigmoid(heatmap) * query_heatmap_score * one_hot(query_labels)`,再對類別維取 max → `[num_proposals]`; +- `bbox_pred = cat([center, height, dim, rot, vel]) ` → `[10, num_proposals]`; +- `label = query_labels[0]` → `[num_proposals]`。 + +`bbox_pred` 這 10 維依序是:`(center_x_feat, center_y_feat, z_gravity, dim0_log, dim1_log, dim2_log, sin, cos, vx, vy)` +——注意 center 還在特徵座標、dim 還是 log 尺度,所以 **一定要經過 bbox_coder 解碼**(§7.4)。 + +--- + +## 7. Stage 4 詳解 — Evaluation loop 逐步拆解 + +### 7.1 `EvaluationOrchestrator.run()` — 決定「評誰、在哪」 +[`evaluation_orchestrator.py`](deployment/runtime/evaluation_orchestrator.py#L57): + +1. 若 `evaluation.enabled=False` 直接跳過。 +2. [`_resolve_model_specs()`](deployment/runtime/evaluation_orchestrator.py#L111):走訪 `evaluation.backends` 每個 backend: + - 只留 `enabled=True` 的; + - [`_resolve_device_for_backend()`](deployment/runtime/evaluation_orchestrator.py#L144) 決定 device(TensorRT 一定要 CUDA,否則覆寫成預設 CUDA 並警告); + - [`ArtifactManager.resolve_artifact()`](deployment/runtime/artifact_manager.py#L53) 找 artifact 路徑(先看註冊過的,再看 `evaluation.backends..model_dir/engine_dir`,再看 fallback); + - artifact 存在才產生一個 `ModelSpec(backend, device, artifact)`。 +3. `num_samples`:`-1` 代表全部(取 `data_loader.num_samples`)。 +4. **對每個 `ModelSpec` 呼叫 `self.evaluator.evaluate(...)`**,把結果存進 `all_results[backend]`,並 `print_results`。 + 任何 backend 失敗都被 `try/except` 包住(記 `error`),`finally` 一定 `clear_cuda_memory()`。 +5. 若不只一個 backend,呼叫 [`_print_cross_backend_comparison()`](deployment/runtime/evaluation_orchestrator.py#L190) 印比較表 + (每個 backend 的 `summarize_for_comparison` 行:mAP/mAPH/latency)。 + +> 在 `deploy_config.py` 中,`pytorch.enabled=False`、`tensorrt.enabled=True`,所以預設只評 TensorRT。 +> 想要 PyTorch↔TensorRT 對照,把 `pytorch.enabled=True`。 + +### 7.2 `BaseEvaluator.evaluate()` — 核心 loop +[`base_evaluator.py`](deployment/evaluation/base_evaluator.py#L123),對「一個 backend」做: + +```python +self._executor.ensure_model_on_device(model.device) # 參考模型搬到目標 device +pipeline = self._executor.create_pipeline(model, model.device) # 建這個 backend 的 pipeline +self.metrics_interface.reset() # 清空 metric buffer + +self._run_warmup(pipeline, data_loader, model, num_warmup, verbose) # 熱身（丟棄結果） + +for idx in range(actual_samples): # actual = min(num_samples, dataset) + sample = data_loader.load_sample(idx) + inference_input = self._executor.prepare_input(sample, data_loader, model.device) + ground_truths = self._parse_ground_truths(sample["ground_truth"]) # ★ 3D 版見 §7.3 + infer_result = pipeline.infer(inference_input.data, metadata=inference_input.metadata) # ★ §7.4 + latencies.append(infer_result.latency_ms) + predictions = self._parse_predictions(infer_result.output) # 直接就是 list[dict] + self._add_to_interface(predictions, ground_truths) # → metrics.add_frame + pipeline.periodic_cleanup(idx) # TRT 每 10 個 sample 清 CUDA cache +# finally: pipeline.cleanup()（釋放 engine/context/buffer） +return self._build_results(latencies, latency_breakdowns, actual_samples) # ★ §7.5 +``` + +重點: +- **warmup**([`_run_warmup`](deployment/evaluation/base_evaluator.py#L191))重用前幾個 sample 跑推論但 **丟棄** 輸出/延遲/metric,只為了暖 GPU/CUDA/TRT 狀態,不影響 `num_samples` 統計。 +- `prepare_input` 由 [`PointCloudBackendExecutor.prepare_input`](deployment/execution/point_cloud_backend_executor.py#L24) 實作: + 只是把 `sample["points"]` + `sample["metainfo"]` 包成 `InferenceInput`(BEVFusion 的 device/資料搬移在 pipeline 內做)。 +- 延遲統計:[`compute_latency_stats`](deployment/evaluation/base_evaluator.py#L219) 算 mean/std/min/max/median; + 若 pipeline 回傳 per-stage `breakdown`,[`_compute_latency_breakdown`](deployment/evaluation/base_evaluator.py#L240) 逐 stage 彙整。 + +### 7.3 GT 解析 — `Detection3DEvaluator._parse_ground_truths` +[`detection_3d_evaluator.py`](deployment/evaluation/detection_3d_evaluator.py#L60):把 `gt_bboxes_3d` / `gt_labels_3d` +轉成 `[{ "bbox_3d": [...7 或 9 維...], "label": int }, ...]`。`_parse_predictions` 則因為 pipeline 已經輸出好 list[dict], +直接原樣回傳。 + +### 7.4 `pipeline.infer()` — 前處理 / 跑模型 / 後處理 +所有 backend 共用 [`BaseInferencePipeline.infer()`](deployment/inference/base_inference_pipeline.py#L138) 的三段式骨架, +每段計時後寫進 `InferenceResult.breakdown`: + +```text +infer(input) = preprocess(input) → run_model(x) → postprocess(y, metadata) + 記 preprocessing_ms / model_ms(+sub-stage) / postprocessing_ms +``` + +BEVFusion 的三段實作在 [`BEVFusionInferencePipeline`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py): + +**(a) preprocess — 體素化** +[`preprocess()`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L70):用 **參考模型的 `pts_voxel_layer`** +把 `points` 變成 `voxels / coors / num_points_per_voxel`。這一步刻意 **放在 ONNX 圖外**——所以不論 PyTorch 或 TRT, +體素化都用同一段 PyTorch code,消除 backend 差異。只支援 hard voxelization(輸出必須是 3-tuple)。 + +**(b) run_model — sparse + dense 兩段接縫** +[`run_model()`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L104) 把模型切成兩個「接縫」,各自計時: +- `run_sparse_encoder(voxels, coors, num_points)` → `lidar_bev`(對應 ONNX `bevfusion_sparse`),記 `sparse_ms`; +- `run_dense(lidar_bev)` → `[bbox_pred, score, label_pred]`(對應 ONNX `bevfusion_dense`),記 `dense_ms`。 + +這兩個接縫是 **抽象方法**,由各 backend 實作: + +| backend | `run_sparse_encoder` | `run_dense` | +| --- | --- | --- | +| PyTorch [`pytorch_inference_pipeline.py`](deployment/projects/bevfusion_l/inference/pytorch_inference_pipeline.py) | 直接跑 `pts_voxel_encoder` + `pts_middle_encoder`(補上 batch 欄) | 跑 `pts_backbone`→`pts_neck`→`_align_lidar_bev_to_head_grid`→`bbox_head`,再 `head_dict_to_detection_outputs` | +| TensorRT split [`tensorrt_inference_pipeline.py`](deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py#L197) | 餵 sparse **引擎**(voxels/coors/num_points → `lidar_bev`),CUDA-event 計時 | 餵 dense **引擎**(`lidar_bev` → 三張量) | +| TensorRT merged | (不切)`_run_merged`:單一全圖引擎跑一次,回報單一 `model_ms` | 同左 | + +TensorRT 版覆寫 `run_model`:split 佈局回報純 GPU 的 `sparse_ms`/`dense_ms`(由 CUDA event 量,見 +[`run_trt_engine`](deployment/inference/tensorrt_runner.py#L112));merged 佈局回報單一 `model_ms`。 +PyTorch 版是 wall-clock 計時,作為參考。 + +**(c) postprocess — bbox 解碼** +[`postprocess()`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L148) 對三張量做: +1. 形狀正規化到 `bbox_pred=[10, num_proposals]`、`score`/`label_pred=[num_proposals]`; +2. 取參考模型的 `bbox_head.bbox_coder`,把 `center/height/dim/rot/vel` **解碼回公制座標** + (`decode(...)` 內部處理 log-dim、特徵座標→世界座標、sin/cos→yaw); +3. 過濾掉 score < 1e-6 的框; +4. 輸出 `[{ "bbox_3d": [cx,cy,z,dx,dy,dz,yaw,vx,vy], "score": float, "label": int }, ...]`。 + +> 為什麼 ONNX 已經有 query 選取還要解碼?因為 ONNX 圖只做到 head 編碼輸出(方便 TRT), +> **座標/尺度的最終解碼刻意留在 PyTorch 後處理**,讓 PyTorch↔ONNX 的比較是在同一個「編碼空間」進行, +> 避免解碼慣例漂移;同時所有 backend 用同一顆 `bbox_coder`,結果可比。 + +### 7.5 累積與計分 — `_add_to_interface` → `_build_results` +- [`_add_to_interface()`](deployment/evaluation/detection_3d_evaluator.py#L82) → `metrics_interface.add_frame(pred, gt)`(§8)。 +- 全部 sample 跑完後,[`_build_results()`](deployment/evaluation/detection_3d_evaluator.py#L87): + - `compute_latency_stats(latencies)`; + - `metrics_interface.compute_metrics()`(觸發真正計分)+ `metrics_interface.summary.to_dict()`; + - 組出 `EvalResultDict`:`mAP_by_mode` / `mAPH_by_mode` / `per_class_ap_by_mode` / `detailed_metrics` / + `latency` /(可選)`latency_breakdown` / `num_samples`。 +- [`print_results()`](deployment/evaluation/detection_3d_evaluator.py#L159):印度量報表、延遲統計、以及 stage-wise 分解 + (`preprocessing_ms` / `model_ms`(或 `sparse_ms`+`dense_ms`)/ `postprocessing_ms`)。 + +--- + +## 8. 度量是怎麼算的 — T4MetricV2 / perception_eval + +evaluation 用的度量跟 **訓練期 T4MetricV2 完全同源**(`autoware_perception_evaluation`),確保「部署後的 mAP」與 +「訓練時的 mAP」可直接對照。 + +### 8.1 度量設定的來源 +[`extract_t4metric_v2_config(model_cfg)`](deployment/metrics/detection_3d_metrics.py#L143):從 `model_cfg.val_evaluator` +(必須是 `T4MetricV2`)抽出 `evaluation_config_dict`(距離門檻、matching 門檻、`min/max_distance` 等)、`frame_id`、 +`critical_object_filter_config`、`frame_pass_fail_config`,包成 `Detection3DMetricsConfig`。度量設定 **來自模型 config,而非模型架構**,所以所有 3D 偵測專案共用這支函式。 + +### 8.2 `Detection3DMetricsInterface`(3D 版) +[`detection_3d_metrics.py`](deployment/metrics/detection_3d_metrics.py#L203): +- 建構時依 `min_distance`/`max_distance`(可為 list)展開成 **多個距離範圍 evaluator** + ([`_resolve_distance_ranges`](deployment/metrics/detection_3d_metrics.py#L234) / [`_create_evaluator_specs`](deployment/metrics/detection_3d_metrics.py#L265)), + 每個範圍一個 spec,key 前綴 `bev_center_-`。 +- [`add_frame(pred, gt)`](deployment/metrics/detection_3d_metrics.py#L362):把 pred / gt dict 透過 + [`_to_dynamic_objects_3d`](deployment/metrics/detection_3d_metrics.py#L292) 轉成 perception_eval 的 `DynamicObject` + (`bbox_3d`→position/orientation(yaw→Quaternion)/shape(length,width,height)/velocity;GT 的 score 固定 1.0,並讀 `num_lidar_pts`), + 再 `_buffer_frame` **只 buffer,不立即計分**。非有限值(NaN/inf)的框會被跳過。 + +### 8.3 共用計分流程(`BaseMetricsInterface`) +[`base_metrics_interface.py`](deployment/metrics/base_metrics_interface.py):Template Method。 +- [`compute_metrics()`](deployment/metrics/base_metrics_interface.py#L181):對每個 evaluator spec **當場建 `PerceptionEvaluationManager` + → 重播所有 buffered frame(`add_frame_result`)→ `get_scene_result()` → 立即釋放**。這樣峰值記憶體與「距離範圍數量」無關。 + 結果扁平化後快取(下次 `add_frame`/`reset` 才失效)。 +- [`detection_base.py`](deployment/metrics/detection_base.py):把 perception_eval 的 `MetricsScore` + ([`_extract_scores`](deployment/metrics/detection_base.py#L79))轉成: + - 扁平 metric dict(`{label}_AP_{mode}_{thr}`、`mAP_{mode}`、以及 3D 才有的 `APH`/`mAPH`); + - 結構化 `DetectionSummary`(`mAP_by_mode` / `mAPH_by_mode` / `per_class_ap_by_mode`)。 +- summary 取「最後(最寬)距離桶」的分數([`_select_summary_score`](deployment/metrics/detection_3d_metrics.py#L412))。 + +> `mode`(matching mode)例如 center-distance-bev、plane-distance 等;`by_mode` 就是「同一組預測用不同 matching 準則」各算一份 mAP。 + +--- + +## 9. Config 如何驅動 evaluation 的 backend 與佈局 + +以 [`deploy_config.py`](deployment/projects/bevfusion_l/config/deploy_config.py) 為例: + +- `evaluation`:`enabled` / `num_samples` / `num_warmup` / `verbose` / `backends`; + `backends.tensorrt.engine_dir` 指向 `work_dir/tensorrt/`(評估時去那裡找 `.engine`)。 +- `components`:宣告 `bevfusion_sparse`(voxels/coors/num_points → `lidar_bev`)與 + `bevfusion_dense`(`lidar_bev` → bbox_pred/score/label_pred),各自的 dtype、`dynamic_axes`、`tensorrt_profile`。 + **這些 I/O 名稱就是 ONNX/engine 綁定的名稱**——執行期靠名字餵資料/取輸出,不靠位置猜。 +- `bevfusion_merge`:`enabled=True` 時,`BEVFusionDeploymentConfig` 會 **衍生** 一個 `bevfusion_merged` component。 + +Config 解析: +- [`BaseDeploymentConfig`](deployment/config/base.py#L34) 解析 devices / components / onnx / export / tensorrt / + evaluation / verification,並在 config 階段就驗證 CUDA(若任何階段用到 TRT)。 +- [`BEVFusionDeploymentConfig`](deployment/projects/bevfusion_l/config/bevfusion_deployment_config.py) 加上 4 個旗標 + (`fuse_spconv_bn` / `spconv_do_sort` / `spconv_fuse_implicit_gemm_relu` / `merge_bevfusion`),並在建構時 + 用 [`add_merged_component`](deployment/projects/bevfusion_l/config/component_layout.py#L48) 把 + merged 全圖從 split pair 衍生出來(sparse 的 inputs + dense 的 outputs),最後 `_validate_components`。 +- [`component_layout.py`](deployment/projects/bevfusion_l/config/component_layout.py):`is_split_components`(有無 sparse+dense)、 + `has_component`、`merge_requested`。 + +**三種佈局如何決定 evaluation 走哪條路**([`BEVFusionExecutor`](deployment/projects/bevfusion_l/evaluation/executor.py) + +[`BEVFusionTensorRTInferencePipeline.__init__`](deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py#L55)): + +| 佈局 | components | evaluation 執行 | +| --- | --- | --- | +| split | sparse + dense(無 merged) | TRT 載 **兩個引擎**,分兩段跑,回報 `sparse_ms`+`dense_ms` | +| split + merge(本 config) | sparse + dense + merged | 若磁碟上有 merged engine → 載 **單一全圖引擎**,回報單一 `model_ms`;否則退回 split | +| merged only | merged | 單一全圖引擎 | + +`BEVFusionExecutor.get_supported_backends` 明講 **只支援 PyTorch 與 TensorRT**; +ONNXRuntime 不能跑 sparse 圖(需要 TensorRT-only 的 Autoware plugin),所以 ONNX 只用於匯出、不用於推論。 + +--- + +## 10. ONNX 這一側:每個部件在做什麼 + +雖然本文主題是 evaluation,但 evaluation 用的 TensorRT engine 是從 ONNX 來的,而且使用者想理解「ONNX 每個部分」。 +以下拆解匯出流程(當 `export.mode` 含 `onnx` 時由 `ExportOrchestrator._export_onnx` 觸發)。 + +### 10.0 匯出的骨架 — 共用 `OnnxExportPipeline` +[`onnx_export_pipeline.py`](deployment/export/pipelines/onnx_export_pipeline.py):model-agnostic,**一個 component 一個 ONNX 檔**。 +`export()` 流程: +1. `sample_extractor.extract_sample(...)` 取一個 tracing 樣本; +2. `component_builder.build_components(model, sample)` 把模型切成 1 或 2 個 component; +3. 對每個 component:`get_onnx_settings(name)` 取 I/O 名稱/dynamic_axes/opset → `ONNXExporter.export`(§10.7)→ + 若有 `post_transforms` 就對匯出的 `.onnx` 依序套用(§10.5); +4. 若有 `finalize` hook,全部匯完後執行(§10.6)。 + +BEVFusion 透過幾個「注入點」表達自己的特性,而 **不 fork 整條 pipeline**(見專案的 export-seam 慣例): +`sample_extractor` + `component_builder` +(component 級的)`post_transforms` +(pipeline 級的)`finalize`。 + +### 10.1 tracing 樣本 — `BEVFusionSampleExtractor` +[`sample_extractor.py`](deployment/projects/bevfusion_l/export/sample_extractor.py)(使用者剛剛打開的檔案): +載入一個點雲樣本 → `model.pts_voxel_layer(points)` 體素化 → 把 coors 從體素層的 `[x,y,z]` +**flip 成 ONNX graph-input 的 `[z,y,x]`**([`voxel_indices_xyz_to_graph_input_zyx`](deployment/projects/bevfusion_l/io/voxel_inputs.py#L65))→ +回傳 typed 的 [`BEVFusionVoxelSample`](deployment/projects/bevfusion_l/io/sample_types.py)(voxels / coors[int32] / num_points)。 +這個樣本只是 **決定 tracing 時的 shape/型別**。 + +### 10.2 切圖 — `BEVFusionComponentBuilder` +[`component_builder.py`](deployment/projects/bevfusion_l/export/component_builder.py) 是一個 **純**「model + 已就緒 sample → components」的步驟:device/dtype/座標由 extractor 負責(§10.1)、`spconv_do_sort` 由 runner 在 load 時設定(§4.2),所以 builder 自己不碰 device/dtype、也沒有全域副作用。 +- split 佈局 → 產出兩個 `ExportableComponent`: + - `bevfusion_sparse`:module = [`BEVFusionSparseWrapper`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L42); + 若 `spconv_fuse_implicit_gemm_relu` 為真,post_transform = ImplicitGemm+ReLU 融合(§10.5b); + - `bevfusion_dense`:先用 `_run_sparse_encoder` 在樣本上跑一次得到 `lidar_bev` 當 tracing 輸入,module = + [`BEVFusionDenseWrapper`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L64),post_transform = TopK 常數化(§10.5a); +- merged 佈局 → 單一 `bevfusion_merged` 全圖**不由 builder 直接匯出**,而是由 [`transforms.py`](deployment/projects/bevfusion_l/export/transforms.py) 的 merge finalize hook 把 split 的 sparse+dense ONNX 組合而成(見下方 split→merge); +- 三個元件走同一個 `_component()` 樣板:tracing 輸入直接取自 typed sample(`_voxel_inputs(sample)` = `sample.voxels/coors/num_points_per_voxel`,不再 `.to(device)`/`.to(int32)`);`name` 取自 `components_cfg.get_component("").name`(與 CenterPoint 同一個 pattern,順帶驗證 component 存在); +- TopK 的常數 K 由 `_num_proposals(model)` 取自**單一來源** `model.bbox_head.num_proposals`; +- LiDAR-only 前提在 model load 時由 `_require_lidar_only_bevfusion` 一次檢查(§4.2),builder 不再重複驗證。 + +> 註:sparse 匯出時 spconv 會印一行良性的 advanced-indexing `UserWarning`,屬正常現象、不影響結果。 + +### 10.3 ONNX wrappers — 把子模組包成固定 I/O 簽名 +[`onnx_models/bevfusion_onnx.py`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py): +- [`normalize_sparse_coors_for_autoware`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L22): + 圖 **輸入** 契約是 `[z,y,x]`(無 batch);wrapper 內把 coors flip 回 `[x,y,z]`、補上 batch 欄、確保 int32 + (符合舊 Autoware ONNX 契約)。座標契約細節見 [`voxel_inputs.py`](deployment/projects/bevfusion_l/io/voxel_inputs.py) 與 doc 25。 +- `BEVFusionSparseWrapper.forward(voxels, coors, num_points)` → `mod.extract_pts_feat(...)` → `lidar_bev`(sparse 分支)。 +- `BEVFusionDenseWrapper.forward(lidar_bev)` → backbone→neck→grid 對齊→bbox_head → `head_dict_to_detection_outputs`(dense 分支)。 +- 全圖 `bevfusion_merged` 沒有對應的 module wrapper:它是 sparse ONNX + dense ONNX 由 merge finalize hook 事後組合而成(§split→merge),而非單獨 trace。 +- 兩個 wrapper 都用同一支 `head_dict_to_detection_outputs`(§6),所以 ONNX 的輸出契約與 PyTorch 完全一致。 + +### 10.4 SparseConv+BN 折疊(匯出前的圖優化) +[`spconv_bn_fusion.py`](deployment/projects/bevfusion_l/export/spconv_bn_fusion.py):在 **模型載入時**(`build_bevfusion_model(fuse_spconv_bn=True)`) +就把 `pts_middle_encoder` 裡每對 `SparseConvolution + BatchNorm1d` 用 spconv 的 eval-mode Conv-BN fold 合併 +(BN 換成 `Identity`)。這 **不是量化**,只是圖優化,讓匯出的 sparse ONNX 是 BN-free、與 runtime 圖一致。 +因為在 load 時就折好,runtime sparse encoder 可直接被 trace,不需要另外建 FP32 shadow encoder。 + +### 10.5 匯出後的 ONNX graph transforms(`post_transforms`) +共用 pipeline 的 [`_apply_post_transforms`](deployment/export/pipelines/onnx_export_pipeline.py#L189) 會 load `.onnx` → +依序套 transform → save 回去。BEVFusion 用兩個: + +**(a) TopK 常數化** — [`fix_topk_constant_k`](deployment/projects/bevfusion_l/export/transforms.py#L31): +`torch.onnx.export` 可能產生動態的 TopK `K`,但 TensorRT 要求 `K` 是常數。這個 transform 把(唯一的)TopK 節點的 +`K` 換成常數 `num_proposals`,並修好輸出 shape。只在 dense 套(merged 全圖由 dense ONNX 帶入這個修正)。 + +**(b) ImplicitGemm + ReLU 融合** — [`fuse_autoware_implicit_gemm_trailing_relu`](deployment/projects/bevfusion_l/export/onnx_fuse_implicit_gemm_activation.py#L85): +TensorRT 不會自動把標準 ONNX `Relu` 融進自訂 op,所以手動把 `autoware.ImplicitGemm → Relu` 的 pattern 折成 +「帶 activation 的 ImplicitGemm」(設 `act_type=kReLU` 並刪掉獨立的 `Relu` 節點)。只在 sparse 且 `spconv_fuse_implicit_gemm_relu=True` 時套。 + +### 10.6 split → merged 的合併(pipeline 級 `finalize`) +[`bevfusion_merge_finalize`](deployment/projects/bevfusion_l/export/transforms.py#L173) → +[`merge_split_sparse_dense_onnx`](deployment/projects/bevfusion_l/export/transforms.py#L68):用 `onnx.compose` 把 +`sparse.onnx` + `dense.onnx` 接成單一 `bevfusion_merged` ONNX(統一 IR/opset、加前綴、用 `io_map` 把 +`sparse/lidar_bev` 接到 `dense/lidar_bev`、把外部 I/O 名稱改回 config 宣告的名字)。當 `merge_bevfusion=True` 才啟用。 + +### 10.7 真正的 `torch.onnx.export` — `ONNXExporter` +[`onnx_exporter.py`](deployment/export/exporters/onnx_exporter.py):`export()` = `_prepare_for_onnx`(套 wrapper、`eval()`)→ +`_do_onnx_export`(在 **私有 staging 目錄** 呼叫 `torch.onnx.export`,含 `opset_version` / `do_constant_folding` / +`input_names` / `output_names` / `dynamic_axes`,再原子性 publish,避免半成品或外部權重檔錯位)→ 可選 `onnxsim` 簡化。 +I/O 名稱與 dynamic_axes 全來自 deploy config 的 `components..io`([`get_onnx_settings`](deployment/config/base.py#L156))。 + +### 10.8 為什麼要 split(sparse / dense)? +| | sparse(`pts_middle_encoder`) | dense(backbone/neck/head) | +| --- | --- | --- | +| ONNX I/O | `voxels,coors,num_points → lidar_bev` | `lidar_bev → bbox_pred,score,label_pred` | +| 主要 op | `autoware::ImplicitGemm`(自訂) | 標準 `Conv/ReLU/Add/TopK/Gather...` | +| TensorRT | **需要自訂 plugin**(`libautoware_tensorrt_plugins.so`) | TRT 原生 | + +split 讓 dense 塔可以走純 TensorRT,只有 sparse 塔需要 Autoware ImplicitGemm plugin;merge 再把兩者接回單圖方便部署。 + +--- + +## 11. TensorRT 執行細節(evaluation 實際餵資料的地方) + +[`tensorrt_inference_pipeline.py`](deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py) + +共用 runner [`tensorrt_runner.py`](deployment/inference/tensorrt_runner.py): + +- `__init__` 先 `load_tensorrt_plugin_libraries`(dlopen spconv plugin)、`init_libnvinfer_plugins`,再依佈局載引擎。 +- **餵 voxel 輸入**:[`_prepare_voxel_inputs`](deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py#L151) + 把 voxels→float32、coors→`[z,y,x]` int32(`voxel_indices_xyz_to_graph_input_zyx`)、num_points→int32 且 `max(_,1)`(避免 mean-pool 除以 0 造成 NaN BEV)。 + [`map_voxel_inputs`](deployment/projects/bevfusion_l/io/voxel_inputs.py#L31) 依引擎宣告的輸入名綁定三個陣列(名稱不符會直接報錯)。 +- **執行 + 計時**:[`run_trt_engine`](deployment/inference/tensorrt_runner.py#L112) 處理所有 dtype/buffer 細節 + (輸入依 binding dtype cast——FP16 引擎關鍵)、用 CUDA event 只框住 `execute_async_v3` → 得到 **純 GPU 時間**。 +- **輸出排序**:[`order_outputs_by_config`](deployment/inference/base_inference_pipeline.py#L65) 依 config 宣告順序回傳 + (ONNX/TRT 回報順序可能任意),確保 `[bbox_pred, score, label_pred]` 對得上後處理。 +- **資源管理**:`GPUResourceMixin` 保證 `cleanup()` 只跑一次;`periodic_cleanup` 每 10 個 sample 清一次 CUDA cache。 + +--- + +## 12. 從輸入到 mAP 的資料型別流(一頁速查) + +```text +dataset[idx] + → SampleData{ points[N,F], metainfo, ground_truth{gt_bboxes_3d, gt_labels_3d, num_lidar_pts} } + │ +prepare_input ─────────────────────▶ InferenceInput{ data=points, metadata=metainfo } + │ +preprocess (pts_voxel_layer) ──────▶ { voxels[M,P,C], coors[M,3](x,y,z), num_points[M] } + │ +run_sparse_encoder ────────────────▶ lidar_bev [B, 256, H, W] （sparse 引擎 / PyTorch spconv） +run_dense ─────────────────────────▶ [ bbox_pred[10,Q], score[Q], label_pred[Q] ]（編碼空間） + │ +postprocess (bbox_coder.decode) ───▶ [ {bbox_3d:[cx,cy,z,dx,dy,dz,yaw,vx,vy], score, label}, ... ]（公制） + │ +_parse_predictions + _parse_ground_truths + │ +metrics.add_frame → DynamicObject（buffer） + │ +compute_metrics（每個距離範圍建 evaluator、重播、算分、釋放） + │ +EvalResultDict{ mAP_by_mode, mAPH_by_mode, per_class_ap_by_mode, latency(, latency_breakdown), num_samples } +``` + +--- + +## 13. 常見疑問(對照原始碼) + +- **Q:為什麼 evaluation 之前一定要 export?** A:即使 `export.mode="none"`,`ExportOrchestrator.run()` 仍會載入 + PyTorch 模型並回填給 executor;PyTorch/TRT 的前處理(體素化)與後處理(解碼)都需要這顆參考模型。 +- **Q:TensorRT 評估時報單一 `model_ms` 還是 `sparse_ms`+`dense_ms`?** A:看磁碟上是否有 merged engine。有 → 單引擎單 `model_ms`;沒有 → split 雙引擎兩段計時。見 + [`BEVFusionTensorRTInferencePipeline.__init__`](deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py#L69)。 +- **Q:為什麼不用 ONNXRuntime 評估?** A:sparse 圖用 `autoware::ImplicitGemm` 自訂 op,只有 TensorRT plugin 有實作, + ORT 沒有。ONNX 只是 PyTorch→TensorRT 的橋。見 [`get_supported_backends`](deployment/projects/bevfusion_l/evaluation/executor.py#L46)。 +- **Q:部署 mAP 為什麼能跟訓練 mAP 對照?** A:度量設定直接抽自 `model_cfg.val_evaluator`(T4MetricV2),且用同一個 + `autoware_perception_evaluation` 引擎與同樣的距離範圍。見 §8。 +- **Q:coors 的座標順序?** A:體素層輸出 `[x,y,z]`;ONNX/TRT 圖輸入契約是 `[z,y,x]`(無 batch),wrapper 內再 flip 回 + `[x,y,z]` 並補 batch。PyTorch 評估直接用 `[batch,x,y,z]`。見 [`voxel_inputs.py`](deployment/projects/bevfusion_l/io/voxel_inputs.py) 與 doc 25。 + +--- + +## 14. 延伸閱讀 +- 專案架構地圖:[`../README.md`](deployment/projects/bevfusion_l/README.md) +- 框架整體:[`../../docs/architecture.md`](deployment/docs/architecture.md) +- coors 契約 / Autoware 對齊:[`25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md`](deployment/projects/bevfusion_l/docs/25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md) +- ScatterND→SECOND trace 差異:[`26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](deployment/projects/bevfusion_l/docs/26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) +- 2.8.x 部署:[`28_README_BEVFUSION_2_8_DEPLOYMENT.md`](deployment/projects/bevfusion_l/docs/28_README_BEVFUSION_2_8_DEPLOYMENT.md) +- ONNX 節點數對齊:[`29_README_ONNX_NODE_COUNT_ALIGNMENT.md`](deployment/projects/bevfusion_l/docs/29_README_ONNX_NODE_COUNT_ALIGNMENT.md) + +> **環境提醒**:ONNX/TensorRT 匯出與評估都在 BEVFusion 部署 Docker 內執行;sparse 的 ImplicitGemm plugin `.so` +> 必須先 build 好、並存在於 `tensorrt_config.plugin_libraries` 指定的路徑。 diff --git a/deployment/projects/bevfusion_l/docs/31_README_MODEL_ARCHITECTURE_AND_SHAPE_WALKTHROUGH.md b/deployment/projects/bevfusion_l/docs/31_README_MODEL_ARCHITECTURE_AND_SHAPE_WALKTHROUGH.md new file mode 100644 index 000000000..41978dc9d --- /dev/null +++ b/deployment/projects/bevfusion_l/docs/31_README_MODEL_ARCHITECTURE_AND_SHAPE_WALKTHROUGH.md @@ -0,0 +1,494 @@ +# BEVFusion-L: End-to-End Model Architecture & Shape Walkthrough + +> **What this document is.** A complete, first-hand trace of the **BEVFusion-L** (LiDAR-only) +> 3D-detection model as it is deployed by the `bevfusion_l` bundle. Every shape and node fact below +> was captured by actually running **one point-cloud sample end-to-end** — through both the native +> **PyTorch** path and the built **TensorRT** engines — inside the `bevfusion-deployment:latest` +> container, and by parsing the exported ONNX graphs. It documents, per stage: the exact input/output +> tensor shapes, what every PyTorch module does, and what every ONNX node does. + +--- + +## 0. How this was produced (reproducible) + +```bash +# Container: bevfusion-deployment:latest (id 173f7f9e…), repo bind-mounted at /workspace, +# data at /workspace/data, spconv plugin at /opt/plugins/libautoware_tensorrt_plugins.so +# The deployment run that built the ONNX + engines: +python -m deployment.cli.main bevfusion_l \ + deployment/projects/bevfusion_l/config/deploy_config.py \ + projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb16_j6gen2_base_120m_t4metric_v2.py +``` + +Artifacts consumed by this walkthrough (already built): + +``` +work_dirs/bevfusion_deployment_2_8/ +├── onnx/ +│ ├── bevfusion_sparse.onnx # 138 nodes — voxel encoder + sparse 3D conv → dense BEV +│ ├── bevfusion_dense.onnx # 242 nodes — SECOND + FPN + TransFusion head +│ └── bevfusion_lidar_fp16_opt.onnx # 380 nodes — merged full graph (= 138 + 242) +└── tensorrt/ + ├── bevfusion_sparse.engine + ├── bevfusion_dense.engine + └── bevfusion_lidar_fp16_opt.engine # merged; this is what the TRT backend actually runs +``` + +The trace/instrumentation scripts used are described in [§11](#11-how-to-reproduce-the-trace). The +sample used is **dataset index 0** of the j6gen2 test split. + +--- + +## 1. TL;DR — the whole pipeline on one sample + +``` +raw LiDAR points [460528, 5] (x, y, z, intensity, time_lag) + │ (A) voxelization — hard voxelization, OUTSIDE the graph + ▼ +voxels [70747, 32, 5] up to 32 points/voxel +coors [70747, 3] voxel index (z,y,x at the graph boundary) +num_points_per_voxel [70747] + │ ══════════ bevfusion_sparse (ONNX / TRT engine) ══════════ + │ (B) voxel encoder (mean-pool + sin/cos Fourier) → [70747, 50] + │ (C) sparse 3D conv encoder (spconv, 21 ImplicitGemm layers) + │ (D) scatter sparse → dense, collapse Z into channels + ▼ +lidar_bev [1, 256, 180, 180] dense BEV feature map + │ ══════════ bevfusion_dense (ONNX / TRT engine) ══════════ + │ (E) SECOND backbone → (F) SECONDFPN neck → [1, 512, 180, 180] + │ (G) TransFusion head: heatmap → top-500 query selection → + │ 1 transformer decoder layer → per-query regression + scoring + ▼ +bbox_pred [10, 500] score [500] label_pred [500] (500 proposals) + │ (H) bbox decode + circle-NMS — OUTSIDE the graph + ▼ +78 detections (PyTorch FP32) / 77 detections (TRT FP16) for sample 0 +``` + +Two boundaries are **outside** the exported graph and live in the Python pipeline: +**(A) voxelization** (`preprocess`) and **(H) box decoding + NMS** (`postprocess`). Everything between +`voxels` and `(bbox_pred, score, label_pred)` is the neural network, exported as ONNX/TensorRT. + +--- + +## 2. Model configuration (resolved values) + +Resolved by following the `_base_` chain from the model config used on the CLI. Sources: +`projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_intensity_120m.py`, +`.../default/models/default_lidar_second_secfpn_120m.py`, and the `_120m.py` base. + +| Parameter | Value | Meaning | +|---|---|---| +| `point_cloud_range` | `[-122.4, -122.4, -3.0, 122.4, 122.4, 5.0]` | x/y/z min–max (metres); 120 m range | +| `voxel_size` | `[0.17, 0.17, 0.2]` | metres per voxel (x, y, z) | +| `grid_size` (`sparse_shape`) | `[1440, 1440, 41]` | voxels along x, y, z = range/voxel_size | +| `sparse_dense_output_shapes` | `[180, 180, 2]` | dense grid after sparse encoder (x, y, z) | +| `out_size_factor` | `8` | BEV feature stride: 1440 / 8 = **180** | +| `max_num_points` | `32` | max points per voxel (hard voxelization) | +| `max_voxels` | `[120000, 160000]` | cap (train / test); actual this frame = 70747 | +| `point feature dim` | `5` | (x, y, z, intensity, time_lag) | +| `class_names` | `[car, truck, bus, bicycle, pedestrian, traffic_cone, barrier]` | **7 classes** | +| `num_proposals` | `500` | queries selected by TopK → detections/frame | +| `num_decoder_layers` | `1` | transformer decoder depth | +| `hidden_channel` | `128` | decoder/query embedding dim | +| `num_heads` | `8` | attention heads (128 / 8 = 16 dims/head) | + +**Full model definition** (`default_lidar_second_secfpn_120m.py`): + +```python +model = dict( + type="BEVFusion", + pts_voxel_encoder = HardSimpleVoxelSinCosEncoder(in_channels=5) # → 5*5*2 = 50 ch + pts_middle_encoder = BEVFusionSparseEncoder( + in_channels=50, sparse_shape=[1440,1440,41], + encoder_channels=((16,16,32),(32,32,64),(64,64,128),(128,128)), + encoder_paddings=((0,0,1),(0,0,1),(0,0,(1,1,0)),(0,0)), + block_type="basicblock") + pts_backbone = SECOND(in_channels=256, out_channels=[128,256], + layer_nums=[5,5], layer_strides=[1,2]) + pts_neck = SECONDFPN(in_channels=[128,256], out_channels=[256,256], + upsample_strides=[1,2]) # concat → 512 ch + bbox_head = BEVFusionHead(in_channels=512, hidden_channel=128, + num_proposals=500, num_decoder_layers=1, + nms_kernel_size=3, num_heads=8, + common_heads=dict(center=[2,2], height=[1,2], + dim=[3,2], rot=[2,2], vel=[2,2]), + bbox_coder=TransFusionBBoxCoder(out_size_factor=8, code_size=10, + score_threshold=[.015,.010,.010,.020,.030,.040,.020]), + test_cfg=dict(nms_type="circle", nms_clusters=[...])) +) +``` + +--- + +## 3. Stage A — Voxelization (`preprocess`, outside the graph) + +`BEVFusionInferencePipeline.preprocess` runs the model's own hard-voxelization layer +(`pts_voxel_layer`) on the CPU/GPU point tensor. It is **not** part of the ONNX graph — the graph's +first input is `voxels`. + +| Tensor | Shape (sample 0) | dtype | Notes | +|---|---|---|---| +| `points` (input) | `[460528, 5]` | float32 | (x, y, z, intensity, time_lag) | +| `voxels` | `[70747, 32, 5]` | float32 | per-voxel point buffer, zero-padded to 32 | +| `coors` | `[70747, 3]` | int32 | voxel grid index. Model-internal is `[x,y,z]`; the **graph input contract is `[z,y,x]`** | +| `num_points_per_voxel` | `[70747]` | int32 | valid points per voxel (1…32) | + +> **Coordinate contract.** At the ONNX/TRT boundary `coors` is `[z, y, x]` (no batch column). The +> wrappers flip it back to `[x, y, z]` and prepend a batch column before spconv — see +> `io/voxel_inputs.py` and `export/onnx_models/bevfusion_onnx.py::normalize_sparse_coors_for_autoware`. +> `num_points_per_voxel` is clamped to `>= 1` before the mean-pool so empty voxels never divide by zero +> (a NaN there poisons the whole dense BEV). + +--- + +## 4. Stage B–D — Sparse branch (`bevfusion_sparse`) + +**Component:** `bevfusion_sparse.onnx` (138 nodes) / `bevfusion_sparse.engine`. +**Signature:** `(voxels, coors, num_points_per_voxel) → lidar_bev [1,256,180,180]`. + +ONNX op histogram: `GetIndicePairsImplicitGemm ×21`, `ImplicitGemm ×21` (the spconv plugin pairs), +`Add ×12`, `Relu ×8`, `Constant ×32`, plus the voxel-encoder (`ReduceSum, Div, Mul, Add, Cos, Sin, +Concat`) and the scatter-to-dense tail (`ScatterElements, Reshape, Transpose`). + +### 4.1 Voxel encoder — `HardSimpleVoxelSinCosEncoder` (nodes 15–32) + +Source: `projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py`. What it computes: + +1. **Mean-pool** each voxel over its valid points: `[70747, 32, 5] → [70747, 5]` + (ONNX `ReduceSum` over the 32-point axis, then `Div` by `num_points_per_voxel`). +2. **Min–max normalize + Fourier expand.** For each of the 5 channels `j` and each frequency + exponent `i ∈ {0..4}`, form `y[n,i,j] = (mean[n,j]−min_j)/(max_j−min_j) · π · 2^i`. + This is folded into a single fused multiply-add `y = bias + scale · mean` (ONNX `Mul`+`Add`), + giving a `[70747, 5, 5]` tensor reshaped to `[70747, 25]`. + Normalization ranges: `min=[−122.4,−122.4,−3,0,0]`, `max=[122.4,122.4,5,255,0.2]`. +3. **sin/cos** concatenation: `concat(cos(y), sin(y)) → [70747, 50]` (ONNX `Cos`, `Sin`, `Concat`). + +Output: **`voxel_features [70747, 50]`** — this is why the sparse encoder's `in_channels = 50`. + +### 4.2 Sparse 3D encoder — `BEVFusionSparseEncoder` (nodes 33–91) + +Submanifold/regular sparse 3D convolutions (spconv), each realized in ONNX as a +`GetIndicePairsImplicitGemm` (builds the gather/scatter rulebook) + `ImplicitGemm` (the actual +conv, with the trailing ReLU **fused into the plugin's `act_type`** — that is why there are only 8 +explicit `Relu` nodes, not 21). Captured spconv tensor shapes `(features, spatial [x,y,z])`: + +| Module | Feature channels | Sparse spatial `[x, y, z]` | Active voxels | +|---|---|---|---| +| `conv_input` (SubMConv3d) | 16 | `[1440, 1440, 41]` | 70747 | +| `encoder_layer1` (basicblock ×2 + downsample) | 32 | `[720, 720, 21]` | 63710 | +| `encoder_layer2` | 64 | `[360, 360, 11]` | 31472 | +| `encoder_layer3` | 128 | `[180, 180, 5]` | 12557 | +| `encoder_layer4` (no downsample) | 128 | `[180, 180, 5]` | 12557 | +| `conv_out` (SparseConv3d, z-stride) | 128 | `[180, 180, 2]` | 9266 | + +Each `encoder_layerN.k` is a **basicblock**: `conv1 → conv2 → Add(residual) → ReLU` (visible in the +ONNX as the `ImplicitGemm, ImplicitGemm, Add, Relu` quartets). Layers 1–3 end with a stride-2 +downsampling sparse conv (`encoder_layerN.2`), halving x/y/z. `conv_out` compresses Z from 5→2. + +### 4.3 Scatter to dense BEV (nodes 92–137) + +The final sparse tensor from `conv_out` (9266 active voxels, 128 ch, grid `[180,180,2]`) must be +turned into a dense `[1,256,180,180]` map for the 2-D backbone. BEVFusion does this with an +**explicit scatter** (`sparse_to_dense` in +`projects/BEVFusion/bevfusion/custom_sparse_conv_tensor.py`) rather than spconv's built-in `.dense()` +— the file header says this exists specifically "to support cleaner ONNX export". The single +`ScatterElements` node in the sparse graph comes from exactly one line of that helper: +`out.scatter(0, scatter_idx, features)` (`torch.Tensor.scatter(dim, index, src)` lowers to ONNX +`ScatterElements`). + +Source → ONNX node mapping (sparse graph nodes 95–137): + +```python +b, h, w, d = idx.unbind(1) # Split + Squeeze×4 (95, 97–103) +linear_idx = ((b*H + h)*W + w)*D + d # flatten (b,h,w,d) → 1-D: Mul/Add (104–112) +out = torch.zeros([num_cells, 128]) # zero dense buffer: ConstantOfShape (120) + # num_cells = 1·180·180·2 = 64800 +scatter_idx = linear_idx.unsqueeze(1).expand(-1, 128) # Unsqueeze + Expand (122, 131) +out = out.scatter(0, scatter_idx, features) # ★ ScatterElements (dim=0) (132) +out.view(1, 180, 180, 2, 128) # Reshape (134) +# then in sparse_encoder.py forward: +.permute(0, 4, 3, 1, 2) # Transpose → [1,128,2,180,180] (135) +.view(1, 256, 180, 180) # Reshape → lidar_bev (137) +``` + +So `ScatterElements` = **write each of the 9266 active voxels' 128-dim feature vector into row +`linear_idx` of a zeroed `[64800, 128]` dense table** (`64800 = 1×180×180×2` cells); empty cells stay +0. It is `ScatterElements` (not `ScatterND`) precisely because the code uses element-wise +`Tensor.scatter` along `dim=0`, a deliberate choice that keeps the graph clean and the node count +stable. This step lives **inside** the sparse graph — `lidar_bev` is its output. + +Finally **Z is folded into the channel dimension** (128 channels × 2 Z-slices = 256): + +``` +Reshape → [1, 180, 180, 2, 128] +Transpose → [1, 128, 2, 180, 180] +Reshape → [1, 256, 180, 180] +``` + +Output: **`lidar_bev [1, 256, 180, 180]`** (float32). PyTorch↔TRT parity on this tensor is within +FP16 tolerance. + +--- + +## 5. Stage E–G — Dense branch (`bevfusion_dense`) + +**Component:** `bevfusion_dense.onnx` (242 nodes) / `bevfusion_dense.engine`. +**Signature:** `lidar_bev [1,256,180,180] → (bbox_pred [10,500], score [500], label_pred [500])`. + +### 5.1 SECOND backbone (ONNX nodes 0–91, interleaved) + +`pts_backbone = SECOND`, two blocks of 5 conv layers each: + +| Block | Stride | Output | +|---|---|---| +| `blocks.0` | 1 | `[1, 128, 180, 180]` | +| `blocks.1` | 2 | `[1, 256, 90, 90]` | + +### 5.2 SECONDFPN neck (nodes 79–95) + +`pts_neck = SECONDFPN` upsamples both scales back to 180×180 and concatenates: + +| Deblock | Op | Output | +|---|---|---| +| `deblocks.0` | conv (stride-1) | `[1, 256, 180, 180]` | +| `deblocks.1` | `ConvTranspose` (×2 up) | `[1, 256, 180, 180]` | +| `Concat` | channel concat | **`[1, 512, 180, 180]`** | + +`_align_lidar_bev_to_head_grid` is a no-op here (neck already at the 180×180 head grid). + +### 5.3 BEVFusion (TransFusion) head — the interesting part + +The head turns the `[1,512,180,180]` BEV map into 500 object queries and regresses a box per query. + +**(a) Shared conv + dense heatmap** (nodes 96–105): +`shared_conv: 512→128` → `[1,128,180,180]`; `heatmap_head` (Conv-BN-ReLU + Conv) → +`dense_heatmap [1, 7, 180, 180]` → `Sigmoid`. 7 = number of classes. + +**(b) Heatmap local-max NMS + TopK query selection** (nodes 106–140): +- A `MaxPool` (kernel `nms_kernel_size=3`) + `Equal`/`Where` keeps only local-peak heatmap cells + (suppresses neighbours). This local-max pooling is applied **only to the "crowded" classes** + `dense_heatmap_pooling_classes = [car, truck, bus, barrier]`; the other classes pass through + un-pooled (see `bevfusion_head.py`). +- Heatmap reshaped `[1,7,180,180] → [1,7,32400] → [1, 226800]` (flatten class × position). +- `TopK` selects the **500** highest scores across *all* classes and positions → + indices `[1,500]`. **The `K=500` constant is baked in** (a post-export transform pins it, see + `export/transforms.py::fix_topk_constant_k`). +- `Div` / `Mod` by 32400 decode each flat index into `(class_id, bev_position)`: + `query_labels [1,500]` and the position used to gather the query's BEV feature and 2-D position. + +**(c) Query & positional embeddings** (nodes 130–152): +- `query_feat`: gather the 128-ch BEV feature at each of the 500 positions → `[1,128,500]`. +- `query_pos`: the (x,y) BEV coordinate of each query → embedded by `self_posembed` + (`PositionEncodingLearned`, Conv1d-BN-ReLU-Conv1d) → `[1,128,500]`. +- `class_encoding`: one-hot class → Conv1d → added into the query feature. +- The **whole BEV** (32400 positions) is embedded once by `cross_posembed` → `[1,128,32400]` to serve + as the cross-attention key positions. + +**(d) Transformer decoder — 1 layer** (`decoder.0`, nodes 153–204): + +| Sub-block | ONNX ops | Query tensor | +|---|---|---| +| **Self-attention** (queries↔queries) | `MatMul` Q/K/V, `Mul`(scale), `MatMul`+`Softmax` (8 heads, `[8,500,500]`), `Gemm` out-proj | `[1,500,128]` | +| `norms.0` | `LayerNormalization` | `[1,500,128]` | +| **Cross-attention** (queries↔BEV) | `MatMul` Q/K/V, `Softmax` `[8,500,32400]`, `Gemm` | `[1,500,128]` | +| `norms.1` | `LayerNormalization` | `[1,500,128]` | +| **FFN** | `MatMul 128→256 → ReLU → MatMul 256→128` | `[1,500,128]` | +| `norms.2` | `LayerNormalization` | `[1,500,128]` | + +Cross-attention is where each object query reads the BEV features (all 32400 cells) to refine itself. + +**(e) Prediction heads — `SeparateHead`** (nodes 205–239): the `[1,128,500]` decoded queries pass +through six Conv1d(128→64→out) branches (`common_heads`), each producing a per-query regression: + +| Head | Out channels | Meaning | +|---|---|---| +| `center` | 2 | BEV offset (feature-map units); **`+= query_pos`** (node 224) | +| `height` | 1 | z of gravity centre (metres) | +| `dim` | 3 | log box size (dx, dy, dz) | +| `rot` | 2 | (sin θ, cos θ) | +| `vel` | 2 | (vx, vy) | +| `heatmap` | 7 | per-query class logits | + +`Concat` (node 239) stacks center+height+dim+rot+vel → **`bbox_pred [10, 500]`**. + +**(f) Scoring** (nodes 137–241, the output contract in `io/head_outputs.py`): + +```python +score = sigmoid(heatmap) * query_heatmap_score * one_hot(query_labels) # [1,7,500] +score = score.max(over classes) # → score [500] +label = query_labels[0] # → label_pred [500] +``` +Implemented in ONNX by `Sigmoid`, `OneHot`, `GatherElements`, `Mul`, `ReduceMax`, `Gather`. + +**Dense outputs (sample 0):** `bbox_pred [10,500] f32`, `score [500] f32`, `label_pred [500] i64`. + +--- + +## 6. Stage H — Postprocess (`postprocess`, outside the graph) + +`BEVFusionInferencePipeline.postprocess` reproduces the reference-eval selection so PyTorch/TRT match +`test.py`. It calls the model's own `TransFusionBBoxCoder.decode(filter=True)` +(`projects/BEVFusion/bevfusion/utils.py`): + +``` +center_x_metric = center_x_feat · out_size_factor(8) · voxel_size_x(0.17) + pc_range_x(−122.4) +center_y_metric = center_y_feat · 8 · 0.17 + (−122.4) +dim = exp(dim_log) # log-size → metres +z_bottom = height − dim_z · 0.5 # gravity centre → bottom centre +yaw = atan2(rot_sin, rot_cos) +box = [x, y, z, dx, dy, dz, yaw, vx, vy] +``` + +Then per-class **score thresholds** (`[.015,.010,.010,.020,.030,.040,.020]`), a `post_center_range` +filter, and **circle-NMS by cluster** (`apply_cluster_nms`; car/truck/bus radius 0.25, others 0.0) +prune the 500 proposals to the final detections. + +**Result for sample 0:** PyTorch → **78** detections, TRT (merged FP16) → **77** detections. Example +(PyTorch): `label 0 (car)`, box `[-8.5, -7.74, 0.05, 4.44, 1.71, 1.56, -0.017, …]`, score 0.92. + +--- + +## 7. ONNX graphs at a glance + +| Graph | Nodes | Initializers | Inputs → Outputs | +|---|---|---|---| +| `bevfusion_sparse` | **138** | — | `voxels[N,32,5], coors[N,3], num_points[N]` → `lidar_bev[1,256,180,180]` | +| `bevfusion_dense` | **242** | 102 | `lidar_bev[1,256,180,180]` → `bbox_pred[10,500], score[500], label_pred[500]` | +| `bevfusion_lidar_fp16_opt` (merged) | **380** | 144 | `voxels/coors/num_points` → `bbox_pred, score, label_pred` | + +The merged graph is exactly `138 + 242 = 380` nodes — it is **composed from the split pair** +post-export (the `bevfusion_merge` finalize hook wires `sparse.lidar_bev → dense.lidar_bev`), not +re-traced. The split pair exists so the sparse (spconv-plugin) and dense parts can be built/profiled +independently; the merged engine is what the TRT backend runs by default. + +`opset 18`, `do_constant_folding=True`. The static `lidar_bev` shape `[1,256,180,180]` lets constant +folding drop the head's dynamic shape-glue so the split and merged node counts line up. + +--- + +## 8. TensorRT engines (bindings) + +Precision **FP16**; the spconv plugin `libautoware_tensorrt_plugins.so` is loaded before +deserialize. Captured bindings: + +**`bevfusion_sparse.engine`** +| I/O | name | shape | dtype | +|---|---|---|---| +| in | `voxels` | `[-1, 32, 5]` | FLOAT | +| in | `coors` | `[-1, 3]` | INT32 | +| in | `num_points_per_voxel` | `[-1]` | INT32 | +| out | `lidar_bev` | `[1, 256, 180, 180]` | FLOAT | + +Dynamic voxel-count profile (`voxels`): min `1`, opt `64000`, max `256000`. + +**`bevfusion_dense.engine`**: in `lidar_bev [1,256,180,180]` → out `bbox_pred [10,500] f32`, +`score [500] f32`, `label_pred [500] i64`. + +**`bevfusion_lidar_fp16_opt.engine`** (merged): in `voxels/coors/num_points` → out +`bbox_pred/score/label_pred` (same as dense). + +> ONNXRuntime is **not** a runtime backend for BEVFusion — the sparse graph needs the TRT-only +> `autoware` spconv plugins, so ONNX is an export/interchange format only. + +--- + +## 9. PyTorch module execution trace (captured, sample 0) + +Abridged from a forward hook on every submodule (211 modules total), in execution order. Shapes are +the live values for this frame. + +``` +pts_voxel_encoder HardSimpleVoxelSinCosEncoder [70747,32,5] → [70747, 50] + +pts_middle_encoder BEVFusionSparseEncoder + conv_input.0 SubMConv3d → sparse(feat[70747,16], grid[1440,1440,41]) + encoder_layer1 → sparse(feat[63710,32], grid[720,720,21]) + encoder_layer2 → sparse(feat[31472,64], grid[360,360,11]) + encoder_layer3 → sparse(feat[12557,128], grid[180,180,5]) + encoder_layer4 → sparse(feat[12557,128], grid[180,180,5]) + conv_out.0 SparseConv3d → sparse(feat[9266,128], grid[180,180,2]) + (scatter→dense) → [1, 256, 180, 180] + +pts_backbone SECOND + blocks.0 → [1, 128, 180, 180] + blocks.1 → [1, 256, 90, 90] +pts_neck SECONDFPN + deblocks.0 → [1, 256, 180, 180] + deblocks.1 (ConvTranspose) → [1, 256, 180, 180] + (concat) → [1, 512, 180, 180] + +bbox_head BEVFusionHead + shared_conv → [1, 128, 180, 180] + heatmap_head → [1, 7, 180, 180] (dense_heatmap) + class_encoding Conv1d → [1, 128, 500] + decoder.0 TransformerDecoderLayer + self_posembed → [1, 128, 500] + cross_posembed → [1, 128, 32400] + self_attn (8 heads) attn map [1,500,500] → [1, 500, 128] + cross_attn (8 heads) attn map [1,500,32400] → [1, 500, 128] + ffn (128→256→128) → [1, 500, 128] + prediction_heads.0 SeparateHead + center → [1,2,500] height → [1,1,500] dim → [1,3,500] + rot → [1,2,500] vel → [1,2,500] heatmap → [1,7,500] + → (bbox_pred [10,500], score [500], label_pred [500]) +``` + +--- + +## 10. Where each concern lives (code map) + +| Concern | File | +|---|---| +| Voxelize / decode / NMS (outside graph) | `inference/bevfusion_inference_pipeline.py` | +| PyTorch backend seams | `inference/pytorch_inference_pipeline.py` | +| TensorRT backend (split/merged) | `inference/tensorrt_inference_pipeline.py` | +| ONNX export wrappers (what each graph computes) | `export/onnx_models/bevfusion_onnx.py` | +| Split→dense component definitions | `export/component_builder.py` | +| Split→merge ONNX finalize | `export/transforms.py` | +| spconv ReLU→ImplicitGemm fusion | `export/onnx_fuse_implicit_gemm_activation.py` | +| Head output contract (score/label) | `io/head_outputs.py` | +| Voxel-input coordinate contract (`[z,y,x]`↔`[x,y,z]`) | `io/voxel_inputs.py` | +| Head-output triple → detection dicts | `io/head_outputs.py`, pipeline `postprocess` | +| Voxel encoder (Fourier) | `projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py` | +| bbox decode | `projects/BEVFusion/bevfusion/utils.py` | +| Model + backbone/neck/head config | `projects/BEVFusion/configs/t4dataset/default/models/default_lidar_second_secfpn_120m.py` | +| Range/voxel/grid config | `projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_intensity_120m.py` | + +--- + +## 11. How to reproduce the trace + +Three small scripts (run inside the container from `/workspace`) produced every number above: + +- **`trace_bevfusion.py`** — builds the model + loads sample 0, registers a forward hook on every + submodule, runs `preprocess → run_sparse_encoder → run_dense → postprocess`, and dumps every + input/output shape to `trace_report.json`. +- **`trace_trt.py`** — deserializes the three engines and prints their I/O bindings + (`trt_bindings.json`). +- **`trace_onnx.py`** — loads each ONNX, runs shape inference, and prints the op histogram + a + node-by-node walk with inferred shapes (`onnx_analysis.json`). +- **`trace_merged.py`** — runs the merged FP16 engine end-to-end and reports the detection count. + +```bash +docker exec awml-bevfusion bash -c 'cd /workspace && python _trace/trace_bevfusion.py' +docker exec awml-bevfusion bash -c 'cd /workspace && python _trace/trace_trt.py' +docker exec awml-bevfusion bash -c 'cd /workspace && python _trace/trace_onnx.py' +docker exec awml-bevfusion bash -c 'cd /workspace && python _trace/trace_merged.py' +``` + +--- + +### Appendix — key numbers for sample 0 + +| Quantity | Value | +|---|---| +| Input points | 460 528 | +| Voxels produced | 70 747 | +| BEV feature map | `[1, 256, 180, 180]` | +| Query proposals | 500 | +| Detections (PyTorch FP32) | 78 | +| Detections (TRT FP16 merged) | 77 | +| Sparse graph nodes | 138 | +| Dense graph nodes | 242 | +| Merged graph nodes | 380 | diff --git a/deployment/projects/bevfusion_l/docs/32_README_MODEL_ARCHITECTURE_detailed.md b/deployment/projects/bevfusion_l/docs/32_README_MODEL_ARCHITECTURE_detailed.md new file mode 100644 index 000000000..64d6aab30 --- /dev/null +++ b/deployment/projects/bevfusion_l/docs/32_README_MODEL_ARCHITECTURE_detailed.md @@ -0,0 +1,2540 @@ +# BEVFusion-L：完整架構、Shape 與 TransFusion Head 詳解 + +> 本文件整理 BEVFusion-L（LiDAR-only）從原始點雲、voxelization、sparse encoder、SECOND/SECONDFPN，到 TransFusion-style detection head、box decode 與 NMS 的完整流程。 +> 數學式使用標準 Markdown LaTeX 語法，可在支援 MathJax / KaTeX 的 Markdown 閱讀器中顯示。 + +--- + +## 1. 模型總覽 + +BEVFusion-L 是 LiDAR-only 版本，因此沒有 camera branch，也沒有 camera-to-BEV pooling。 + +整體流程： + +```text +Raw Point Cloud + │ + ▼ +Hard Voxelization + │ + ▼ +Voxel Feature Encoder + │ + ▼ +Sparse 3D Encoder + │ + ▼ +Dense LiDAR BEV + │ + ▼ +SECOND 2D Backbone + │ + ▼ +SECONDFPN + │ + ▼ +TransFusion-style Head + │ + ▼ +500 Proposals + │ + ▼ +Decode + Circle NMS + │ + ▼ +Final 3D Detections +``` + +此版本只有 LiDAR feature： + +$$ +F_{\text{BEV}} = F_{\text{LiDAR}} +$$ + +--- + +# 2. 輸入點雲 + +單一 sample 的 LiDAR point cloud： + +$$ +P \in \mathbb{R}^{460528 \times 5} +$$ + +每個 point： + +$$ +p_i = (x, y, z, \text{intensity}, \text{time lag}) +$$ + +實際 shape： + +```text +points = [460528, 5] +``` + +Point cloud range： + +$$ +x,y \in [-122.4, 122.4] +$$ + +$$ +z \in [-3.0, 5.0] +$$ + +Voxel size： + +$$ +(\Delta x, \Delta y, \Delta z) += +(0.17, 0.17, 0.2) +$$ + +因此原始 voxel grid 約為： + +$$ +1440 \times 1440 \times 41 +$$ + +因為： + +$$ +\frac{244.8}{0.17} = 1440 +$$ + +--- + +# 3. Stage A：Voxelization + +Voxelization 在 ONNX / TensorRT graph 之外執行。 + +輸入： + +```text +points [460528, 5] +``` + +輸出： + +```text +voxels [70747, 32, 5] +coors [70747, 3] +num_points_per_voxel [70747] +``` + +各維度含義： + +- `70747`：這一幀產生的 non-empty voxel 數量。 +- `32`：每個 voxel 最多保留 32 個 points。 +- `5`：每個 point 的 feature dimension。 +- `num_points_per_voxel[i]`：第 `i` 個 voxel 實際包含的有效 point 數量。 + +因此： + +$$ +\texttt{num\_points\_per\_voxel.shape} = [70747] +$$ + +但每個元素的值滿足： + +$$ +1 \leq \texttt{num\_points\_per\_voxel}[i] \leq 32 +$$ + +例如： + +```text +num_points_per_voxel = [3, 1, 32, 7, ...] +``` + +代表： + +| Voxel | Buffer 容量 | 實際有效點數 | +|---|---:|---:| +| voxel 0 | 32 | 3 | +| voxel 1 | 32 | 1 | +| voxel 2 | 32 | 32 | +| voxel 3 | 32 | 7 | + +Voxel encoder 做平均時使用： + +$$ +f_i += +\frac{ +\sum_{j=0}^{31} V_{i,j} +}{ +\texttt{num\_points\_per\_voxel}[i] +} +$$ + +padding 部分通常是 0，因此可以固定對 32 個位置求和，但分母必須是實際有效點數，而不是固定除以 32。 + +--- + +## 3.1 稀疏程度 + +完整 dense voxel grid 有： + +$$ +1440 \times 1440 \times 41 += +85,017,600 +$$ + +個 cells，但 active voxels 只有： + +$$ +70,747 +$$ + +Active ratio： + +$$ +\frac{70,747}{85,017,600} +\approx 0.083\% +$$ + +這也是 sparse convolution 必要的原因。 + +--- + +# 4. Stage B：Voxel Feature Encoder + +Voxel encoder： + +```text +HardSimpleVoxelSinCosEncoder +``` + +輸入： + +$$ +[70747, 32, 5] +$$ + +輸出： + +$$ +[70747, 50] +$$ + +--- + +## 4.1 Mean pooling + +假設第 $n$ 個 voxel 有 $m_n$ 個有效 points： + +$$ +V_n = +\{p_{n,1}, p_{n,2}, \ldots, p_{n,m_n}\} +$$ + +每個 feature channel 做平均： + +$$ +\bar{p}_n += +\frac{1}{m_n} +\sum_{j=1}^{m_n} p_{n,j} +$$ + +Shape： + +$$ +[70747, 32, 5] +\rightarrow +[70747, 5] +$$ + +得到： + +$$ +\bar{p}_n = +[ +\bar{x}, +\bar{y}, +\bar{z}, +\overline{\text{intensity}}, +\overline{\text{time lag}} +] +$$ + +--- + +## 4.2 Sin/Cos Fourier encoding + +每個原始 channel 使用 5 個頻率： + +$$ +2^i, +\qquad i \in \{0,1,2,3,4\} +$$ + +先 normalize： + +$$ +u_j += +\frac{\bar{p}_j - \min_j} +{\max_j - \min_j} +$$ + +再乘： + +$$ +\pi 2^i +$$ + +得到： + +$$ +y_{i,j} += +u_j \pi 2^i +$$ + +共有： + +$$ +5 \text{ channels} +\times +5 \text{ frequencies} += +25 +$$ + +個值。 + +再計算： + +$$ +[\cos(y), \sin(y)] +$$ + +因此： + +$$ +25 \times 2 = 50 +$$ + +最後： + +```text +voxel_features [70747, 50] +``` + +完整 shape 流程： + +$$ +\boxed{ +[70747,32,5] +\rightarrow +[70747,5] +\rightarrow +[70747,25] +\rightarrow +[70747,50] +} +$$ + +--- + +# 5. Stage C：Sparse 3D Encoder + +輸入： + +```text +features [70747, 50] +coordinates [70747, 4] +spatial grid [1440, 1440, 41] +``` + +實際 shape 演化： + +| Stage | Active voxels | Channels | Sparse spatial shape | +|---|---:|---:|---| +| Input | 70,747 | 50 | $1440 \times 1440 \times 41$ | +| `conv_input` | 70,747 | 16 | $1440 \times 1440 \times 41$ | +| Layer 1 | 63,710 | 32 | $720 \times 720 \times 21$ | +| Layer 2 | 31,472 | 64 | $360 \times 360 \times 11$ | +| Layer 3 | 12,557 | 128 | $180 \times 180 \times 5$ | +| Layer 4 | 12,557 | 128 | $180 \times 180 \times 5$ | +| `conv_out` | 9,266 | 128 | $180 \times 180 \times 2$ | + +在 $x,y$ 方向總 downsample 8 倍： + +$$ +1440 \rightarrow 720 \rightarrow 360 \rightarrow 180 +$$ + +$$ +\frac{1440}{8} = 180 +$$ + +高度方向： + +$$ +41 \rightarrow 21 \rightarrow 11 \rightarrow 5 \rightarrow 2 +$$ + +--- + +## 5.1 Sparse convolution 的概念 + +每一層在 TensorRT 中大致拆成： + +```text +GetIndicePairsImplicitGemm + │ + ▼ +ImplicitGemm +``` + +`GetIndicePairsImplicitGemm` 建立 sparse input-output mapping。 + +`ImplicitGemm` 執行真正的 convolution。 + +概念上： + +$$ +y_j += +\sum_{k \in \mathcal{K}} +W_k x_{M(j,k)} +$$ + +其中： + +- $j$：output active voxel。 +- $k$：kernel offset。 +- $M(j,k)$：對應的 input active voxel。 +- $W_k$：該 kernel offset 的權重。 + +--- + +# 6. Stage D：Sparse to Dense BEV + +Sparse encoder 最後輸出： + +```text +active features [9266, 128] +spatial shape [180, 180, 2] +``` + +完整 dense cells： + +$$ +180 \times 180 \times 2 += +64800 +$$ + +建立： + +```text +dense table [64800, 128] +``` + +初始全部為 0，再用 `ScatterElements` 把 9,266 個 active features 寫入對應位置： + +$$ +D[\text{linear index}_i] = f_i +$$ + +空位置保持 0。 + +Shape 變化： + +$$ +[64800,128] +$$ + +reshape： + +$$ +[1,180,180,2,128] +$$ + +transpose： + +$$ +[1,128,2,180,180] +$$ + +把 $Z=2$ 合併進 channel： + +$$ +[1,128,2,180,180] +\rightarrow +[1,256,180,180] +$$ + +因此： + +$$ +\boxed{ +F_{\text{LiDAR BEV}} +\in +\mathbb{R}^{1 \times 256 \times 180 \times 180} +} +$$ + +其中： + +$$ +256 += +128 \text{ feature channels} +\times +2 \text{ height slices} +$$ + +--- + +# 7. Stage E：SECOND 2D Backbone + +輸入： + +```text +lidar_bev [1,256,180,180] +``` + +Block 0： + +$$ +[1,256,180,180] +\rightarrow +[1,128,180,180] +$$ + +Block 1： + +$$ +[1,128,180,180] +\rightarrow +[1,256,90,90] +$$ + +因此 backbone 產生兩個尺度： + +$$ +F_1 +\in +\mathbb{R}^{1 \times 128 \times 180 \times 180} +$$ + +$$ +F_2 +\in +\mathbb{R}^{1 \times 256 \times 90 \times 90} +$$ + +--- + +# 8. Stage F：SECONDFPN + +第一個 feature： + +$$ +[1,128,180,180] +\rightarrow +[1,256,180,180] +$$ + +第二個 feature： + +$$ +[1,256,90,90] +\xrightarrow{\text{ConvTranspose}} +[1,256,180,180] +$$ + +Channel concatenate： + +$$ +[1,256,180,180] +\oplus +[1,256,180,180] +$$ + +得到： + +$$ +\boxed{ +F_{\text{neck}} +\in +\mathbb{R}^{1 \times 512 \times 180 \times 180} +} +$$ + +--- + +# 9. Detection Head 總覽 + +Head 輸入： + +$$ +F_{\text{neck}} +\in +\mathbb{R}^{1 \times 512 \times 180 \times 180} +$$ + +輸出： + +```text +bbox_pred [10,500] +score [500] +label_pred [500] +``` + +流程： + +```text +Shared Conv + │ + ▼ +Dense Class Heatmap + │ + ▼ +Local Maximum Filtering + │ + ▼ +Top-500 Query Selection + │ + ▼ +Query Feature Initialization + │ + ▼ +Transformer Decoder + │ + ▼ +Separate Prediction Heads + │ + ▼ +500 Boxes + Scores + Labels +``` + +--- + +# 10. Shared BEV Feature + +Shared convolution： + +$$ +[1,512,180,180] +\rightarrow +[1,128,180,180] +$$ + +記為： + +$$ +F_s +\in +\mathbb{R}^{1 \times 128 \times 180 \times 180} +$$ + +每個 BEV cell 有一個 128 維 feature： + +$$ +f_{x,y} +\in +\mathbb{R}^{128} +$$ + +總 cell 數： + +$$ +180 \times 180 = 32400 +$$ + +可以理解為： + +```text +32,400 個 BEV 位置 +每個位置有一個 128 維描述向量 +``` + +--- + +# 11. Dense Heatmap + +模型有 7 個類別： + +```text +0 car +1 truck +2 bus +3 bicycle +4 pedestrian +5 traffic_cone +6 barrier +``` + +Heatmap logits： + +$$ +H_{\text{logit}} +\in +\mathbb{R}^{1 \times 7 \times 180 \times 180} +$$ + +經 sigmoid： + +$$ +H = \sigma(H_{\text{logit}}) +$$ + +其中： + +$$ +H[c,y,x] +$$ + +表示： + +> 模型認為 BEV 位置 $(x,y)$ 是類別 $c$ 的物體中心的可能性。 + +例如： + +```text +H[car, 80, 90] = 0.92 +H[truck, 80, 90] = 0.08 +H[bus, 80, 90] = 0.03 +``` + +--- + +# 12. Local-Max Filtering + +一個物體可能讓相鄰多個 cell 都有高分，例如： + +$$ +\begin{bmatrix} +0.62 & 0.75 & 0.68 \\ +0.71 & 0.92 & 0.74 \\ +0.65 & 0.78 & 0.69 +\end{bmatrix} +$$ + +使用 $3 \times 3$ max-pooling： + +$$ +M = \operatorname{MaxPool}_{3 \times 3}(H) +$$ + +只保留局部最大值： + +$$ +H_{\text{peak}} += +\begin{cases} +H, & H = M \\ +0, & H \neq M +\end{cases} +$$ + +結果： + +$$ +\begin{bmatrix} +0 & 0 & 0 \\ +0 & 0.92 & 0 \\ +0 & 0 & 0 +\end{bmatrix} +$$ + +此模型只對以下 crowded classes 做 local pooling： + +```text +car +truck +bus +barrier +``` + +這一步只是 proposal suppression，不是最終 NMS。 + +--- + +# 13. Flatten Heatmap + +原始 heatmap： + +$$ +[1,7,180,180] +$$ + +每個 class 的空間 flatten： + +$$ +[180,180] +\rightarrow +[32400] +$$ + +因此： + +$$ +[1,7,180,180] +\rightarrow +[1,7,32400] +$$ + +對應線性位置： + +$$ +p = y \times 180 + x +$$ + +反解： + +$$ +x = p \bmod 180 +$$ + +$$ +y = +\left\lfloor +\frac{p}{180} +\right\rfloor +$$ + +再把 class 與 position 合併： + +$$ +7 \times 32400 = 226800 +$$ + +所以： + +$$ +[1,7,32400] +\rightarrow +[1,226800] +$$ + +這 226,800 個元素代表所有： + +$$ +(\text{class}, \text{BEV position}) +$$ + +組合。 + +--- + +# 14. Top-500 Query Selection + +TopK 選的是： + +> 500 個最高分的「類別 + 位置」組合。 + +輸出： + +```text +topk_score [1,500] +topk_index [1,500] +``` + +對 flattened index $i$： + +$$ +\text{class id} += +\left\lfloor +\frac{i}{32400} +\right\rfloor +$$ + +$$ +\text{position id} += +i \bmod 32400 +$$ + +再解出： + +$$ +x = \text{position id} \bmod 180 +$$ + +$$ +y = +\left\lfloor +\frac{\text{position id}}{180} +\right\rfloor +$$ + +--- + +## 14.1 實際例子 + +假設： + +$$ +i = 97110 +$$ + +Class： + +$$ +c += +\left\lfloor +\frac{97110}{32400} +\right\rfloor += +2 +$$ + +Class 2 是 bus。 + +Position： + +$$ +p += +97110 \bmod 32400 += +32310 +$$ + +$$ +x += +32310 \bmod 180 += +90 +$$ + +$$ +y += +\left\lfloor +\frac{32310}{180} +\right\rfloor += +179 +$$ + +因此： + +```text +class = bus +BEV position = (x=90, y=179) +``` + +這只表示： + +> Bus heatmap 認為位置 $(90,179)$ 很可能是 bus center，因此將它選為一個 object query proposal。 + +--- + +# 15. 建立 Object Query + +Top-K 找到 500 個位置後，回到 shared BEV feature： + +$$ +F_s +\in +\mathbb{R}^{1 \times 128 \times 180 \times 180} +$$ + +將 spatial flatten： + +$$ +F_s^{\text{flat}} +\in +\mathbb{R}^{1 \times 128 \times 32400} +$$ + +對 500 個位置 gather： + +$$ +Q_{\text{feat}} +\in +\mathbb{R}^{1 \times 128 \times 500} +$$ + +單一 query： + +$$ +q_i += +F_s[:,y_i,x_i] +$$ + +$$ +q_i +\in +\mathbb{R}^{128} +$$ + +Transformer 常轉置為： + +$$ +Q +\in +\mathbb{R}^{1 \times 500 \times 128} +$$ + +--- + +## 15.1 為什麼不能只使用 heatmap score？ + +Heatmap score 只有一個 scalar，例如： + +$$ +0.92 +$$ + +它只表示「這裡可能有物體」。 + +但 box prediction 還需要： + +- 長度。 +- 寬度。 +- 高度。 +- 朝向。 +- 速度。 +- 類別細節。 + +因此需要取回該位置完整的 128 維 BEV feature。 + +--- + +# 16. Query Position Embedding + +每個 query 有一個 grid position： + +$$ +p_i = (x_i,y_i) +$$ + +使用 learned position embedding： + +$$ +e_i^{\text{pos}} += +\phi_{\text{self-pos}}(x_i,y_i) +$$ + +輸出： + +$$ +e_i^{\text{pos}} +\in +\mathbb{R}^{128} +$$ + +所有 query 的 position embedding： + +$$ +E_{\text{query-pos}} +\in +\mathbb{R}^{1 \times 128 \times 500} +$$ + +這讓 transformer 知道： + +> 每個 query 在 BEV 空間中的位置。 + +Position embedding 使用的是 feature-grid coordinate： + +$$ +x,y \in [0,179] +$$ + +此時尚未轉換成 metric coordinate。 + +--- + +# 17. Class Embedding + +每個 query 是從某個 class heatmap 選出的。 + +例如： + +```text +query 0 → car +query 1 → truck +query 2 → pedestrian +``` + +先轉成 one-hot： + +$$ +o_i \in \mathbb{R}^{7} +$$ + +例如 car： + +$$ +o_{\text{car}} += +[1,0,0,0,0,0,0] +$$ + +再經 learned projection： + +$$ +e_i^{\text{class}} += +W_{\text{class}} o_i +$$ + +$$ +e_i^{\text{class}} +\in +\mathbb{R}^{128} +$$ + +加入 query content： + +$$ +q_i^0 += +q_i + e_i^{\text{class}} +$$ + +因此 query 帶有： + +1. 該位置的 BEV content。 +2. 自己在 BEV 中的位置。 +3. 初始 class prior。 + +--- + +# 18. BEV Memory + +初始 query 只 gather 一個中心 cell： + +$$ +q_i = F_s(x_i,y_i) +$$ + +但一個物體通常跨越多個 cells。 + +Head grid 每 cell 約對應： + +$$ +8 \times 0.17 = 1.36 \text{ m} +$$ + +一台長 4.5 m 的車大約跨越： + +$$ +\frac{4.5}{1.36} +\approx +3.3 +$$ + +個 cells。 + +因此 transformer 保留完整 BEV feature 作為 memory。 + +原始： + +$$ +F_s +\in +\mathbb{R}^{1 \times 128 \times 180 \times 180} +$$ + +Flatten： + +$$ +M +\in +\mathbb{R}^{1 \times 128 \times 32400} +$$ + +Transformer 格式： + +$$ +M +\in +\mathbb{R}^{1 \times 32400 \times 128} +$$ + +總共有： + +$$ +32400 +$$ + +個 BEV memory tokens。 + +--- + +## 18.1 BEV Position Embedding + +每個 BEV memory token 也有位置： + +$$ +e_j^{\text{BEV-pos}} += +\phi_{\text{cross-pos}}(x_j,y_j) +$$ + +輸出： + +$$ +E_{\text{BEV-pos}} +\in +\mathbb{R}^{1 \times 32400 \times 128} +$$ + +因此 cross-attention 能同時理解： + +- memory feature 的內容。 +- memory feature 在 BEV 中的位置。 + +--- + +# 19. Transformer Decoder + +模型設定： + +```text +num_decoder_layers = 1 +num_proposals = 500 +hidden_channel = 128 +num_heads = 8 +``` + +每個 attention head 的 dimension： + +$$ +d_h += +\frac{128}{8} += +16 +$$ + +Transformer decoder 包含： + +1. Self-attention。 +2. Cross-attention。 +3. Feed-forward network。 +4. Residual connection。 +5. Layer normalization。 + +--- + +# 20. Self-Attention + +Self-attention 讓 500 個 object queries 彼此交換資訊。 + +輸入： + +$$ +Q +\in +\mathbb{R}^{500 \times 128} +$$ + +線性投影： + +$$ +Q_s = QW_Q +$$ + +$$ +K_s = QW_K +$$ + +$$ +V_s = QW_V +$$ + +每個 head： + +$$ +Q_s^{(h)}, +K_s^{(h)}, +V_s^{(h)} +\in +\mathbb{R}^{500 \times 16} +$$ + +Attention matrix： + +$$ +A_{\text{self}}^{(h)} += +\operatorname{softmax} +\left( +\frac{ +Q_s^{(h)} K_s^{(h)\top} +}{ +\sqrt{16} +} +\right) +$$ + +Shape： + +$$ +[500,16] +\times +[16,500] += +[500,500] +$$ + +8 個 heads： + +$$ +[8,500,500] +$$ + +第 $(i,j)$ 個值代表： + +> Query $i$ 在更新自己時，要參考 query $j$ 多少。 + +--- + +## 20.1 Self-Attention 範例 + +假設只有 3 個 queries： + +```text +q0：car proposal at (50,50) +q1：car proposal at (51,50) +q2：pedestrian proposal at (100,120) +``` + +Attention 可能是： + +$$ +A_{\text{self}} += +\begin{bmatrix} +0.55 & 0.40 & 0.05 \\ +0.42 & 0.53 & 0.05 \\ +0.05 & 0.05 & 0.90 +\end{bmatrix} +$$ + +$q_0$ 和 $q_1$： + +- 距離近。 +- feature 相似。 +- class 相同。 + +因此彼此 attention 較強。 + +Self-attention 不會直接刪掉重複 query，它只是讓 proposals 知道其他 proposals 的存在。 + +後續模型可能因此： + +- 降低其中一個 query 的分數。 +- 修改其 box regression。 +- 配合 matching 與 NMS 減少重複。 + +--- + +# 21. Cross-Attention + +Cross-attention 讓每個 object query 回頭查看完整 BEV memory。 + +Query： + +$$ +Q +\in +\mathbb{R}^{500 \times 128} +$$ + +BEV memory： + +$$ +M +\in +\mathbb{R}^{32400 \times 128} +$$ + +每個 head： + +$$ +Q_c^{(h)} +\in +\mathbb{R}^{500 \times 16} +$$ + +$$ +K_c^{(h)}, +V_c^{(h)} +\in +\mathbb{R}^{32400 \times 16} +$$ + +Attention matrix： + +$$ +A_{\text{cross}}^{(h)} += +\operatorname{softmax} +\left( +\frac{ +Q_c^{(h)} K_c^{(h)\top} +}{ +\sqrt{16} +} +\right) +$$ + +Shape： + +$$ +[500,16] +\times +[16,32400] += +[500,32400] +$$ + +8 heads： + +$$ +[8,500,32400] +$$ + +每一列表示： + +> 某個 query 對完整 32,400 個 BEV cells 的關注分布。 + +--- + +## 21.1 單一 Query 的 Cross-Attention + +假設 car query 位於： + +$$ +(x=90,y=80) +$$ + +它可能關注： + +```text +(90,80) center 0.20 +(89,80) left side 0.12 +(91,80) right side 0.14 +(90,79) front 0.18 +(90,81) rear 0.16 +other locations 0.20 total +``` + +更新後： + +$$ +q_i^{\text{cross}} += +\sum_{j=1}^{32400} +a_{ij} v_j +$$ + +其中： + +$$ +\sum_{j=1}^{32400} a_{ij} = 1 +$$ + +因此 query 不只使用中心 cell，而是使用整張 BEV 的加權摘要。 + +--- + +## 21.2 Cross-Attention 的矩陣乘法 + +Attention score： + +$$ +[500,16] +\times +[16,32400] += +[500,32400] +$$ + +再乘 value： + +$$ +[500,32400] +\times +[32400,16] += +[500,16] +$$ + +8 個 heads 各輸出 16 維，concatenate： + +$$ +8 \times 16 = 128 +$$ + +最終回到： + +$$ +[500,128] +$$ + +--- + +# 22. Self-Attention 與 Cross-Attention 比較 + +| 項目 | Self-attention | Cross-attention | +|---|---|---| +| Query 來源 | 500 object queries | 500 object queries | +| Key/Value 來源 | 500 object queries | 32,400 BEV tokens | +| 每 head attention shape | `[500,500]` | `[500,32400]` | +| 主要作用 | 理解 proposals 間關係 | 從完整場景讀取資訊 | +| 問題 | 其他候選跟我有什麼關係？ | BEV 哪些位置支持我的判斷？ | + +簡化： + +```text +Self-attention： +物體候選彼此看。 + +Cross-attention： +物體候選回頭看場景。 +``` + +--- + +# 23. Residual Connection 與 LayerNorm + +Self-attention： + +$$ +Q_1 += +\operatorname{LN} +\left( +Q_0 ++ +\operatorname{SelfAttn}(Q_0) +\right) +$$ + +代表： + +```text +更新後 query += +原始 query ++ +從其他 queries 取得的新資訊 +``` + +Cross-attention： + +$$ +Q_2 += +\operatorname{LN} +\left( +Q_1 ++ +\operatorname{CrossAttn}(Q_1,M) +\right) +$$ + +代表： + +```text +query 原有資訊 ++ +從完整 BEV 讀到的新資訊 +``` + +Residual connection 保留原始 proposal feature。 + +LayerNorm 提升訓練與數值穩定性。 + +--- + +# 24. Feed-Forward Network + +FFN： + +$$ +128 \rightarrow 256 \rightarrow 128 +$$ + +對每個 query 獨立執行： + +$$ +q_i' += +W_2 +\operatorname{ReLU} +(W_1 q_i + b_1) ++ +b_2 +$$ + +FFN 不負責 query 間資訊交換；資訊交換已經由 attention 完成。 + +FFN 的作用是： + +> 將 attention 收集到的資訊重新轉換成更適合 box prediction 的 representation。 + +完整 decoder： + +$$ +Q_1 += +\operatorname{LN} +\left( +Q_0 ++ +\operatorname{SelfAttn}(Q_0) +\right) +$$ + +$$ +Q_2 += +\operatorname{LN} +\left( +Q_1 ++ +\operatorname{CrossAttn}(Q_1,M) +\right) +$$ + +$$ +Q_3 += +\operatorname{LN} +\left( +Q_2 ++ +\operatorname{FFN}(Q_2) +\right) +$$ + +最終： + +$$ +Q_3 +\in +\mathbb{R}^{1 \times 500 \times 128} +$$ + +--- + +# 25. Transformer Decoder 的白話解釋 + +初始 query： + +```text +我是在位置 (90,80) 找到的 car 候選。 +這個位置的 BEV feature 是一個 128 維向量。 +``` + +Self-attention 後： + +```text +我發現附近還有另一個很相似的 car 候選， +遠方則有其他不相關候選。 +``` + +Cross-attention 後： + +```text +我查看完整 BEV， +發現中心前後左右都有支持一台車的 feature， +因此能推斷物體範圍與方向。 +``` + +FFN 後： + +```text +我把收集到的資訊整理成適合預測 +center、dimension、rotation、velocity 的 128 維特徵。 +``` + +--- + +# 26. Separate Prediction Heads + +Decoded query： + +$$ +Q_3 +\in +\mathbb{R}^{1 \times 500 \times 128} +$$ + +通常轉回： + +$$ +[1,128,500] +$$ + +每個 branch 大致是： + +```text +Conv1d 128 → 64 +ReLU +Conv1d 64 → output channels +``` + +輸出： + +| Branch | Shape | 含義 | +|---|---|---| +| `center` | `[1,2,500]` | BEV center | +| `height` | `[1,1,500]` | gravity-center z | +| `dim` | `[1,3,500]` | log box dimensions | +| `rot` | `[1,2,500]` | $(\sin\theta,\cos\theta)$ | +| `vel` | `[1,2,500]` | $(v_x,v_y)$ | +| `heatmap` | `[1,7,500]` | refined class logits | + +Regression channels： + +$$ +2 + 1 + 3 + 2 + 2 = 10 +$$ + +所以： + +$$ +bbox_{\text{pred}} +\in +\mathbb{R}^{10 \times 500} +$$ + +單一 query： + +$$ +\hat{b}_i += +[ +c_x, +c_y, +z, +\log d_x, +\log d_y, +\log d_z, +\sin\theta, +\cos\theta, +v_x, +v_y +] +$$ + +--- + +# 27. Center Prediction + +Center branch 預測： + +$$ +\Delta c_i += +(\Delta x_i,\Delta y_i) +$$ + +再加回 query position： + +$$ +c_i^{\text{feat}} += +p_i ++ +\Delta c_i +$$ + +例如： + +$$ +p_i = (80,100) +$$ + +$$ +\Delta c_i = (0.25,-0.30) +$$ + +則： + +$$ +c_i^{\text{feat}} += +(80.25,99.70) +$$ + +此時仍是 feature-map coordinate。 + +--- + +# 28. Score 計算 + +Head 有兩套分類資訊。 + +Proposal score： + +$$ +s_i^{\text{proposal}} +$$ + +來自 dense heatmap Top-K。 + +Transformer 後的 query classification： + +$$ +L_i \in \mathbb{R}^{7} +$$ + +經 sigmoid： + +$$ +s_{i,c}^{\text{query}} += +\sigma(L_{i,c}) +$$ + +Query 有 initial class： + +$$ +c_i^{\text{proposal}} +$$ + +One-hot： + +$$ +o_{i,c} += +\begin{cases} +1, & c = c_i^{\text{proposal}} \\ +0, & \text{otherwise} +\end{cases} +$$ + +最終： + +$$ +s_{i,c}^{\text{final}} += +s_{i,c}^{\text{query}} +\cdot +s_i^{\text{proposal}} +\cdot +o_{i,c} +$$ + +最後： + +$$ +score_i += +\max_c s_{i,c}^{\text{final}} +$$ + +Label： + +$$ +label_i += +c_i^{\text{proposal}} +$$ + +--- + +# 29. Feature Coordinate 轉 Metric Coordinate + +Feature map stride： + +$$ +\text{out size factor} = 8 +$$ + +Voxel size： + +$$ +0.17 \text{ m} +$$ + +每個 head BEV cell 對應： + +$$ +8 \times 0.17 += +1.36 \text{ m} +$$ + +Center decode： + +$$ +x_{\text{metric}} += +x_{\text{feature}} +\cdot +8 +\cdot +0.17 ++ +x_{\min} +$$ + +$$ +y_{\text{metric}} += +y_{\text{feature}} +\cdot +8 +\cdot +0.17 ++ +y_{\min} +$$ + +其中： + +$$ +x_{\min} += +y_{\min} += +-122.4 +$$ + +--- + +## 29.1 Center Decode 範例 + +假設： + +$$ +c_x^{\text{feat}} = 80.25 +$$ + +$$ +c_y^{\text{feat}} = 99.70 +$$ + +則： + +$$ +x_{\text{metric}} += +80.25 \times 1.36 - 122.4 += +-13.26 \text{ m} +$$ + +$$ +y_{\text{metric}} += +99.70 \times 1.36 - 122.4 += +13.192 \text{ m} +$$ + +--- + +# 30. Dimension Decode + +Head 預測 log dimension： + +$$ +[ +\log d_x, +\log d_y, +\log d_z +] +$$ + +實際尺寸： + +$$ +d_x = \exp(\log d_x) +$$ + +$$ +d_y = \exp(\log d_y) +$$ + +$$ +d_z = \exp(\log d_z) +$$ + +例如： + +$$ +[1.435,0.531,0.445] +$$ + +則： + +$$ +d_x \approx 4.20 +$$ + +$$ +d_y \approx 1.70 +$$ + +$$ +d_z \approx 1.56 +$$ + +--- + +# 31. Rotation Decode + +Head 預測： + +$$ +(r_s,r_c) += +(\sin\theta,\cos\theta) +$$ + +Yaw： + +$$ +\theta += +\operatorname{atan2}(r_s,r_c) +$$ + +例如： + +$$ +r_s = -0.017 +$$ + +$$ +r_c = 0.999 +$$ + +則： + +$$ +\theta +\approx +-0.017 \text{ rad} +$$ + +--- + +# 32. Height Decode + +Head 預測 gravity center： + +$$ +z_g +$$ + +最終 box 使用 bottom center： + +$$ +z_{\text{bottom}} += +z_g +- +\frac{d_z}{2} +$$ + +例如： + +$$ +z_g = 0.83 +$$ + +$$ +d_z = 1.56 +$$ + +則： + +$$ +z_{\text{bottom}} += +0.83 - 0.78 += +0.05 +$$ + +--- + +# 33. 單一 Proposal 完整數值例子 + +假設： + +```text +query position = (80,100) +proposal class = car +proposal score = 0.96 +``` + +Center offset： + +$$ +\Delta x = 0.25 +$$ + +$$ +\Delta y = -0.30 +$$ + +因此： + +$$ +c_x = 80.25 +$$ + +$$ +c_y = 99.70 +$$ + +Height： + +$$ +z_g = 0.83 +$$ + +Dimension logits： + +$$ +[1.435,0.531,0.445] +$$ + +Decode： + +$$ +[d_x,d_y,d_z] += +[4.20,1.70,1.56] +$$ + +Rotation： + +$$ +[\sin\theta,\cos\theta] += +[-0.017,0.999] +$$ + +$$ +\theta += +-0.017 +$$ + +Velocity： + +$$ +[v_x,v_y] += +[2.1,0.1] +$$ + +Query classification car score： + +$$ +s_{\text{car}}^{\text{query}} += +0.958 +$$ + +Final score： + +$$ +s^{\text{final}} += +0.958 \times 0.96 +\approx +0.920 +$$ + +Metric center： + +$$ +x += +80.25 \times 1.36 - 122.4 += +-13.26 +$$ + +$$ +y += +99.70 \times 1.36 - 122.4 += +13.19 +$$ + +Bottom z： + +$$ +z += +0.83 - \frac{1.56}{2} += +0.05 +$$ + +最終 box： + +$$ +\boxed{ +[ +-13.26, +13.19, +0.05, +4.20, +1.70, +1.56, +-0.017, +2.1, +0.1 +] +} +$$ + +Score： + +$$ +0.920 +$$ + +Label： + +```text +car +``` + +--- + +# 34. Postprocess + +Network 固定輸出 500 個 proposals。 + +後處理： + +```text +Box Decode + │ + ▼ +Per-Class Score Threshold + │ + ▼ +Post-Center-Range Filtering + │ + ▼ +Circle NMS + │ + ▼ +Final Detections +``` + +Class thresholds： + +```text +car 0.015 +truck 0.010 +bus 0.010 +bicycle 0.020 +pedestrian 0.030 +traffic_cone 0.040 +barrier 0.020 +``` + +Sample 0： + +```text +PyTorch FP32 → 78 detections +TensorRT FP16 → 77 detections +``` + +--- + +# 35. 完整 Shape Walkthrough + +```text +Raw points +[460528,5] + │ + ▼ +Hard voxelization + │ + ├── voxels [70747,32,5] + ├── coors [70747,3] + └── num_points [70747] + │ + ▼ +Mean pooling +[70747,5] + │ + ▼ +Sin/Cos Fourier encoding +[70747,50] + │ + ▼ +Sparse 3D encoder +70747×16 +→ 63710×32 +→ 31472×64 +→ 12557×128 +→ 9266×128 + │ + ▼ +Scatter + Z collapse +[1,256,180,180] + │ + ▼ +SECOND backbone +├── [1,128,180,180] +└── [1,256,90,90] + │ + ▼ +SECONDFPN +[1,512,180,180] + │ + ▼ +Shared convolution +[1,128,180,180] + │ + ├───────────────────────────────────┐ + │ │ + ▼ │ +Dense heatmap │ +[1,7,180,180] │ + │ │ + ▼ │ +Local peak filtering │ + │ │ + ▼ │ +Flatten │ +[1,226800] │ + │ │ + ▼ │ +TopK 500 │ + │ │ + ├── class [1,500] │ + ├── position [1,500] │ + └── proposal score [1,500] │ + │ + ┌──────── gather features ──────────┘ + ▼ +Initial queries +[1,128,500] + │ + ├── class embedding [1,128,500] + └── position embedding [1,128,500] + │ + ▼ +Transformer format +[1,500,128] + │ + ▼ +Self-attention +8 × [500,500] + │ + ▼ +Cross-attention +8 × [500,32400] + │ + ▼ +FFN +128 → 256 → 128 + │ + ▼ +Decoded queries +[1,500,128] + │ + ├── center [1,2,500] + ├── height [1,1,500] + ├── dim [1,3,500] + ├── rot [1,2,500] + ├── vel [1,2,500] + └── heatmap [1,7,500] + │ + ▼ +bbox_pred [10,500] +score [500] +label [500] + │ + ▼ +Decode + threshold + circle NMS + │ + ▼ +77–78 final detections +``` + +--- + +# 36. 最精簡數學表示 + +Voxel 與 sparse encoder： + +$$ +F_{3D} += +E_{\text{sparse}} +\left( +E_{\text{voxel}} +( +\operatorname{Voxelize}(P) +) +\right) +$$ + +轉成 BEV： + +$$ +F_{\text{BEV}} += +\operatorname{CollapseZ} +\left( +\operatorname{ScatterDense}(F_{3D}) +\right) +$$ + +2D backbone： + +$$ +F_{\text{neck}} += +\operatorname{SECONDFPN} +\left( +\operatorname{SECOND}(F_{\text{BEV}}) +\right) +$$ + +Dense proposal： + +$$ +H += +\sigma +\left( +H_{\text{dense}}(F_{\text{neck}}) +\right) +$$ + +Query selection： + +$$ +(c_i,p_i) += +\operatorname{DecodeIndex} +\left( +\operatorname{TopK}(H,500) +\right) +$$ + +Query initialization： + +$$ +q_i^0 += +F_s(p_i) ++ +E_{\text{class}}(c_i) +$$ + +Transformer： + +$$ +Q_1 += +\operatorname{LN} +\left( +Q_0 ++ +\operatorname{SelfAttn}(Q_0) +\right) +$$ + +$$ +Q_2 += +\operatorname{LN} +\left( +Q_1 ++ +\operatorname{CrossAttn}(Q_1,M) +\right) +$$ + +$$ +Q_3 += +\operatorname{LN} +\left( +Q_2 ++ +\operatorname{FFN}(Q_2) +\right) +$$ + +Box prediction： + +$$ +B += +H_{\text{box}}(Q_3) +$$ + +最終 detection： + +$$ +\mathcal{D} += +\operatorname{NMS} +\left( +\operatorname{Decode}(B) +\right) +$$ + +--- + +# 37. 核心理解 + +整個 head 並不是： + +```text +TopK 找到位置 +→ 直接輸出 box +``` + +而是： + +```text +TopK： +從 226,800 個 class-position 候選中， +選出 500 個粗略 object proposals。 + +Gather： +取出每個 proposal 所在 cell 的 128 維內容。 + +Class / Position Encoding： +告訴 query 自己可能是哪一類、位於哪裡。 + +Self-Attention： +讓 500 個 proposals 彼此交換資訊。 + +Cross-Attention： +讓每個 proposal 從完整 32,400-cell BEV 場景收集資訊。 + +FFN： +把收集到的資訊轉成適合 3D box regression 的 feature。 + +Separate Heads： +預測 center、height、dimension、rotation、velocity 與 class score。 +``` + +最核心的一句話： + +> Dense heatmap 負責找候選；object queries 負責表示候選；self-attention 負責候選之間的關係；cross-attention 負責從完整 BEV 中收集資訊；prediction heads 負責輸出最終 3D box。 diff --git a/deployment/projects/bevfusion_l/docs/33_README_MODEL_ARCHITECTURE_code_mapping.md b/deployment/projects/bevfusion_l/docs/33_README_MODEL_ARCHITECTURE_code_mapping.md new file mode 100644 index 000000000..5c8b97bf7 --- /dev/null +++ b/deployment/projects/bevfusion_l/docs/33_README_MODEL_ARCHITECTURE_code_mapping.md @@ -0,0 +1,392 @@ +# BEVFusion-L：操作 ↔ 程式碼對應表 + +> 本文件是 [`32_README_MODEL_ARCHITECTURE_detailed.md`](deployment/projects/bevfusion_l/docs/32_README_MODEL_ARCHITECTURE_detailed.md) 的**程式碼對照版**。 +> 章節編號與 32 號文件一一對應，每個操作都標出實際檔案與行號，讓你能同時看到「數學/流程」與「真正跑的程式碼」。 +> +> 程式碼分屬兩層： +> - **模型層（PyTorch model）**：`projects/BEVFusion/bevfusion/` — 定義 voxel encoder、sparse encoder、SECOND/FPN、TransFusion head、bbox coder。這是 train 與 eval 都會跑的原始模型。 +> - **部署層（deployment）**：`deployment/projects/bevfusion_l/` — 把模型切成 `sparse` / `dense` 兩個可匯出的元件（ONNX/TensorRT），並在 graph 外做 voxelization 與 decode+NMS。 +> +> **核心對應關係**（先記住這張圖，其餘都是細節）： +> +> | 文件流程 | 部署層 wrapper / pipeline | 模型層真正的運算 | +> |---|---|---| +> | Voxelization（graph 外） | `preprocess()` → `pts_voxel_layer` | `bevfusion.py: voxelize()` | +> | `bevfusion_sparse` 元件 | `BEVFusionSparseWrapper.forward` | `extract_pts_feat` = voxel_encoder + middle_encoder | +> | `bevfusion_dense` 元件 | `BEVFusionDenseWrapper.forward` | `pts_backbone` + `pts_neck` + `bbox_head` | +> | 輸出打包成 (bbox,score,label) | `head_dict_to_detection_outputs` | — | +> | Decode + threshold + NMS（graph 外） | `postprocess()` | `TransFusionBBoxCoder.decode` + `apply_cluster_nms` | + +--- + +## §1 模型總覽 — 兩個可匯出元件的切分 + +整份「Raw Point Cloud → Final 3D Detections」流程，在部署層被切成兩個 wrapper，這是理解一切的骨架： + +- Sparse 分支（voxels/coors/num_points → BEV feature map）： + [`bevfusion_onnx.py:45-64`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L45-L64) `BEVFusionSparseWrapper` +- Dense 分支（BEV feature map → bbox/score/label）： + [`bevfusion_onnx.py:67-83`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L67-L83) `BEVFusionDenseWrapper` + +執行期（PyTorch / TensorRT）用同一條 pipeline 依序跑這兩段並各自計時： +[`bevfusion_inference_pipeline.py:101-143`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L101-L143) `run_model()`（`run_sparse_encoder` → `run_dense`）。 + +--- + +## §2–3 輸入點雲 & Voxelization（在 graph 之外） + +文件重點：`points [460528,5]` → `voxels [70747,32,5]` / `coors [70747,3]` / `num_points_per_voxel [70747]`。 + +**部署層**：voxelization 刻意放在 ONNX graph 之外，由 pipeline 的 `preprocess()` 做： + +[`bevfusion_inference_pipeline.py:67-99`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L67-L99) +```python +def preprocess(self, points): + points_tensor = self.to_device_tensor(points).float() + with torch.no_grad(): + voxel_output = self.pytorch_model.pts_voxel_layer(points_tensor) # ← hard voxelization + voxels, coors, num_points_per_voxel = voxel_output # 只支援 hard voxelization + return {"voxels": voxels, "coors": coors, "num_points_per_voxel": num_points_per_voxel} +``` + +**模型層**：`pts_voxel_layer` 背後的實作在 +[`bevfusion.py:262`](projects/BEVFusion/bevfusion/bevfusion.py#L262) `voxelize()`（`@torch.no_grad()`）。 + +> **文件對應的 §3 平均公式**（除以「實際有效點數」而非固定 32）就是下面 §4 voxel encoder 裡 `features.sum(dim=1) / num_points` 這一行 —— padding 位置是 0，但分母用 `num_points`。 + +`coors` 的座標軸順序（`[x,y,z]` ↔ graph 的 `[z,y,x]`）契約全部集中在： +[`voxel_inputs.py`](deployment/projects/bevfusion_l/io/voxel_inputs.py#L1-L72)（`graph_input_zyx_to_model_indices_xyz` 等），並在 sparse wrapper 進 spconv 前翻轉： +[`bevfusion_onnx.py:25-42`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L25-L42) `normalize_sparse_coors_for_autoware()`。 + +--- + +## §4 Stage B：Voxel Feature Encoder（mean + Sin/Cos Fourier） + +文件重點：`[70747,32,5] → [70747,5] → [70747,25] → [70747,50]`。 + +**模型層**：[`bevfusion_voxel_encoder.py:11-77`](projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py#L11-L77) `HardSimpleVoxelSinCosEncoder`。 + +§4.1 Mean pooling（分母是有效點數，對應文件 §3 的平均式）： +[`bevfusion_voxel_encoder.py:63-65`](projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py#L63-L65) +```python +voxel_mean_features = (features.sum(dim=1, keepdim=False) + / num_points.type_as(features).view(-1, 1)).contiguous() # [N,32,5] -> [N,5] +``` + +§4.2 Sin/Cos Fourier encoding（`u_j·π·2^i` 被折疊成 `scale·x + bias`，一個 FMA 完成）： +- 常數預先算好：[`bevfusion_voxel_encoder.py:39-46`](projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py#L39-L46) + ```python + exponents = (2 ** torch.arange(0, self.in_channels)).float() # 頻率 2^i, i∈{0..C-1} + alpha = (torch.pi * exponents).unsqueeze(0) # π·2^i ← 文件的 π 2^i + scale = alpha / beta # beta = max-min（normalize） + bias = -(alpha * min_norm_values.unsqueeze(1)) / beta + ``` +- 前向：[`bevfusion_voxel_encoder.py:69-74`](projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py#L69-L74) + ```python + y = torch.addcmul(self.exponent_bias, self.exponent_scale, voxel_mean_features.unsqueeze(-1)) # [N,C,C] + y = y.reshape(-1, self.in_channels * self.in_channels) # [N, C*C]=25 + voxel_fourier_features = torch.cat([torch.cos(y), torch.sin(y)], dim=1) # [N, C*C*2]=50 + ``` + +> 精確說明文件的「5 channels × 5 frequencies = 25」：程式碼是 `in_channels × in_channels`。頻率個數 = `in_channels`，此模型 `in_channels=5`（x,y,z,intensity,time_lag），所以 5×5=25，×2(cos,sin)=50。 + +--- + +## §5 Stage C：Sparse 3D Encoder + +文件重點：active voxels 隨層數下降、xy 下採樣 8 倍、高度 41→2。 + +**模型層**：[`sparse_encoder.py:22`](projects/BEVFusion/bevfusion/sparse_encoder.py#L22) `BEVFusionSparseEncoder`，前向： +[`sparse_encoder.py:122-152`](projects/BEVFusion/bevfusion/sparse_encoder.py#L122-L152) +```python +input_sp_tensor = SparseConvTensor(voxel_features, coors, self.sparse_shape, batch_size) +x = self.conv_input(input_sp_tensor) # conv_input: 50 -> 16 ch +for encoder_layer in self.encoder_layers: # Layer 1..4：文件那張表的每一列 + x = encoder_layer(x) + encode_features.append(x) +out = self.conv_out(encode_features[-1]) # conv_out: 高度 5 -> 2 +``` + +§5.1 文件說的 `GetIndicePairsImplicitGemm` → `ImplicitGemm` 是 spconv 每一層在 TensorRT 中的拆解；匯出後還會把後面的 ReLU 折進 plugin 的 `act_type`： +[`onnx_fuse_implicit_gemm_activation.py`](deployment/projects/bevfusion_l/export/onnx_fuse_implicit_gemm_activation.py#L1)（`fuse_autoware_implicit_gemm_trailing_relu`）。 +匯出前的 SparseConv+BN 融合：[`spconv_bn_fusion.py`](deployment/projects/bevfusion_l/export/spconv_bn_fusion.py#L1)。 + +**這一整段的入口**是 sparse wrapper： +[`bevfusion_onnx.py:52-64`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L52-L64) → `self.mod.extract_pts_feat(...)` → +[`bevfusion.py:200-216`](projects/BEVFusion/bevfusion/bevfusion.py#L200-L216) +```python +def extract_pts_feat(self, feats, coords, sizes, points=None): + ... + feats = self.pts_voxel_encoder(feats, sizes, coords) # §4 + x = self.pts_middle_encoder(feats, coords, batch_size) # §5 + §6 + return x +``` + +--- + +## §6 Stage D：Sparse → Dense BEV（ScatterElements + Z collapse） + +文件重點：`[9266,128]` 稀疏 → dense `[64800,128]` → reshape/transpose → 把 Z=2 併進 channel → `[1,256,180,180]`。 + +**模型層**：就在 sparse encoder `forward` 的尾巴： +[`sparse_encoder.py:154-161`](projects/BEVFusion/bevfusion/sparse_encoder.py#L154-L161) +```python +spatial_features = sparse_to_dense(out, batch_size, self.dense_output_shapes, self.output_channels) # scatter +spatial_features = spatial_features.permute(0, 4, 3, 1, 2).contiguous() # → [1,C,Z,H,W] +spatial_features = spatial_features.view(batch_size, + self.output_channels * self.dense_output_shapes[2], # 128 * 2 = 256 ← Z 併進 channel + self.dense_output_shapes[0], self.dense_output_shapes[1]) # [1,256,180,180] +``` +- `sparse_to_dense`（文件的 `ScatterElements`，空位補 0）定義於 + [`custom_sparse_conv_tensor.py`](projects/BEVFusion/bevfusion/custom_sparse_conv_tensor.py#L1)。 + +--- + +## §7–8 Stage E/F：SECOND 2D Backbone + SECONDFPN + +文件重點：backbone 產生 `[1,128,180,180]` 與 `[1,256,90,90]`；FPN 上採樣後 concat 成 `[1,512,180,180]`。 + +**部署層**（dense wrapper 前半段）： +[`bevfusion_onnx.py:74-80`](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L74-L80) +```python +def forward(self, lidar_bev): + x = lidar_bev + if self.mod.pts_backbone is not None: x = self.mod.pts_backbone(x) # §7 SECOND + if self.mod.pts_neck is not None: x = self.mod.pts_neck(x) # §8 SECONDFPN + x = self.mod._align_lidar_bev_to_head_grid(x) # 嚴格檢查 H,W == 180 +``` +**模型層**：`pts_backbone` / `pts_neck` 由 config 建立（SECOND / SECONDFPN，來自 mmdet3d）： +[`bevfusion.py:73-74`](projects/BEVFusion/bevfusion/bevfusion.py#L73-L74)。 +grid 對齊檢查（確保 BEV 尺寸符合 head 的 `grid_size // out_size_factor = 1440//8 = 180`）： +[`bevfusion.py:218-259`](projects/BEVFusion/bevfusion/bevfusion.py#L218-L259) `_align_lidar_bev_to_head_grid`。 + +> 註：`projects/BEVFusion/bevfusion/bevfusion_necks.py` 內另有 `GeneralizedLSSFPN`（camera 分支用）；LiDAR-only 這條由 config 指定的 neck 為主，形狀契約以文件 §8 為準。 + +--- + +## §9–10 Detection Head 總覽 + Shared BEV Feature + +Head 全部集中在一個 forward： +[`bevfusion_head.py:261-381`](projects/BEVFusion/bevfusion/bevfusion_head.py#L261-L381) `forward_single`（入口 `forward` 在 +[`bevfusion_head.py:383-397`](projects/BEVFusion/bevfusion/bevfusion_head.py#L383-L397)）。 + +§10 Shared convolution `[1,512,180,180] → [1,128,180,180]`： +[`bevfusion_head.py:269`](projects/BEVFusion/bevfusion/bevfusion_head.py#L269) +```python +fusion_feat = self.shared_conv(inputs) +fusion_feat_flatten = fusion_feat.view(-1, self.share_conv_out_channels, self.spatial_dim) # [B,128,32400] +``` + +--- + +## §11 Dense Heatmap + +`[1,512..] → heatmap_head → [1,7,180,180]`，再 sigmoid： +[`bevfusion_head.py:280-282`](projects/BEVFusion/bevfusion/bevfusion_head.py#L280-L282) +```python +dense_heatmap = self.heatmap_head(fusion_feat.float()) # heatmap_head 定義於 :132 +heatmap = dense_heatmap.detach().sigmoid() # H = σ(logits) +``` + +--- + +## §12 Local-Max Filtering（只對 crowded classes） + +3×3 max-pool 保留局部極大，且只對特定類別做（文件的 car/truck/bus/barrier）： +[`bevfusion_head.py:283-314`](projects/BEVFusion/bevfusion/bevfusion_head.py#L283-L314) +```python +if self.dense_heatmap_pooling_class_indices is not None: + selected_heatmap = heatmap[:, self.dense_heatmap_pooling_class_indices, :, :] + local_max_inner = F.max_pool2d(selected_heatmap, kernel_size=self.nms_kernel_size, stride=1, padding=0) + local_max = F.pad(local_max_inner, (pad,pad,pad,pad), value=0.0) # 還原空間尺寸 + if self.dense_heatmap_exclude_pooling_classes: # 不做 pooling 的類別直接放回 + excluded_local_max = heatmap[:, self.dense_heatmap_exclude_pooling_classes, :, :] + local_max = torch.cat([local_max, excluded_local_max], dim=1)[:, self.local_concat_class_remapping, :, :] +else: + local_max = heatmap +heatmap = heatmap * (heatmap == local_max) # 只留 H==M 的 peak（文件的 H_peak） +``` + +--- + +## §13 Flatten Heatmap `[1,7,180,180] → [1,226800]` + +[`bevfusion_head.py:316-319`](projects/BEVFusion/bevfusion/bevfusion_head.py#L316-L319) +```python +heatmap = heatmap.view(-1, self.num_classes, self.spatial_dim) # [1,7,32400] +flattened_heatmap = heatmap.view(-1, self.num_classes * self.spatial_dim) # [1, 7*32400=226800] +``` + +--- + +## §14 Top-500 Query Selection + index 反解 class/position + +[`bevfusion_head.py:322-327`](projects/BEVFusion/bevfusion/bevfusion_head.py#L322-L327) +```python +_, top_proposals = torch.topk(flattened_heatmap, k=self.num_proposals, dim=-1, largest=True, sorted=False) +top_proposals_class = top_proposals // self.spatial_dim # ← 文件 i // 32400 +top_proposals_index = top_proposals % self.spatial_dim # ← 文件 i % 32400（BEV 位置） +``` +（`position id → (x,y)` 的 `x=p%180, y=p//180` 反解對應到後面 `bev_pos` 的索引。） + +--- + +## §15 建立 Object Query（gather BEV feature） + +從 shared feature 對 500 個位置 gather 出 128 維 query content： +[`bevfusion_head.py:328-332`](projects/BEVFusion/bevfusion/bevfusion_head.py#L328-L332) +```python +query_feat = fusion_feat_flatten.gather( + index=top_proposals_index[:, None, :].expand(-1, self.share_conv_out_channels, -1), dim=-1) # [B,128,500] +self.query_labels = top_proposals_class +``` + +--- + +## §16 Query Position Embedding & §17 Class Embedding + +§17 Class embedding（one-hot → 1×1 conv → 加到 query content）： +[`bevfusion_head.py:335-337`](projects/BEVFusion/bevfusion/bevfusion_head.py#L335-L337) +```python +one_hot = F.one_hot(top_proposals_class, num_classes=self.num_classes).permute(0, 2, 1) +query_cat_encoding = self.class_encoding(one_hot.float()) # class_encoding = nn.Conv1d(num_classes,128,1) :133 +query_feat += query_cat_encoding +``` +§16 Query position（grid 座標，尚未轉 metric）： +[`bevfusion_head.py:340`](projects/BEVFusion/bevfusion/bevfusion_head.py#L340) +```python +query_pos = self.bev_pos.squeeze(0)[top_proposals_index] # 取 500 個位置的 (x,y) +``` +> 文件 §16 的「learned position embedding `φ_self-pos`」實作在 decoder layer 內的 `self_posembed`（見 §20），`bev_pos` 只是提供每個 query 的 (x,y) grid 座標。 + +--- + +## §18–25 Transformer Decoder（Self-Attn + Cross-Attn + FFN） + +文件 §18 的「BEV memory」= 完整 `fusion_feat_flatten` 當 key/value；§19–25 的 self/cross/FFN 都在一個 decoder layer 裡： + +驅動迴圈（`num_decoder_layers=1`）： +[`bevfusion_head.py:345-358`](projects/BEVFusion/bevfusion/bevfusion_head.py#L345-L358) +```python +for i in range(self.num_decoder_layers): + query_feat = self.decoder[i]( + query_feat, key=fusion_feat_flatten, # ← key/value = 32400 個 BEV token（§18 memory） + query_pos=query_pos, key_pos=self.bev_pos) # §16 self-pos / §18.1 cross-pos + res_layer = self.prediction_heads[i](query_feat) # §26 separate heads + res_layer["center"] = res_layer["center"] + query_pos.permute(0, 2, 1) # §27 center += query_pos +``` + +Decoder layer 定義（Self-Attn / Cross-Attn 就在這）： +[`transformer.py:26-104`](projects/BEVFusion/bevfusion/transformer.py#L26-L104) `TransformerDecoderLayer` +```python +self.self_posembed = PositionEncodingLearned(**pos_encoding_cfg) # §16 φ_self-pos +self.cross_posembed = PositionEncodingLearned(**pos_encoding_cfg) # §18.1 φ_cross-pos +... +query = self.self_attn(...) # §20 self-attention（500×500） +query = self.cross_attn(...) # §21 cross-attention（500×32400） +# 之後 FFN + residual + LayerNorm 由基底 DetrTransformerDecoderLayer 提供 # §23,§24 +``` +- position embedding MLP：[`transformer.py:7-24`](projects/BEVFusion/bevfusion/transformer.py#L7-L24) `PositionEncodingLearned`。 +- §23 residual + LayerNorm、§24 FFN(128→256→128)：繼承自 mmdet 的 `DetrTransformerDecoderLayer`。 + +--- + +## §26 Separate Prediction Heads + +每個 branch 是 Conv1d→ReLU→Conv1d，輸出 center/height/dim/rot/vel/heatmap： +[`bevfusion_head.py:353`](projects/BEVFusion/bevfusion/bevfusion_head.py#L353) `self.prediction_heads[i](query_feat)` +（`prediction_heads` 在 `__init__` 依 `common_heads` 建立）。 +§14 選到的 proposal 分數（`query_heatmap_score`）在此回填： +[`bevfusion_head.py:360-363`](projects/BEVFusion/bevfusion/bevfusion_head.py#L360-L363)。 + +--- + +## §27–28 Center 修正 & Score 計算 + +§27 center 加回 query position：見上 [`bevfusion_head.py:354`](projects/BEVFusion/bevfusion/bevfusion_head.py#L354)。 + +§28 最終分數 `s_query · s_proposal · one_hot`，取 max —— 部署層與 reference eval 用**同一個函式**確保一致： +[`head_outputs.py:15-35`](deployment/projects/bevfusion_l/io/head_outputs.py#L15-L35) `head_dict_to_detection_outputs` +```python +score = outputs["heatmap"].sigmoid() # s_query +one_hot = F.one_hot(outputs["query_labels"], num_classes=score.size(1)).permute(0, 2, 1) +score = score * outputs["query_heatmap_score"] * one_hot # ×s_proposal ×one_hot +score = score[0].max(dim=0)[0] # max_c +bbox_pred = torch.cat([center, height, dim, rot, vel], dim=0) # [10, num_proposals] +return bbox_pred, score, query_labels[0] +``` +> 這正是 §26 表格中 `bbox_pred [10,500] / score [500] / label [500]` 的產生點，也是 ONNX graph 的輸出契約。 +> 原模型 eval 端的等價邏輯在 [`bevfusion_head.py:404-422`](projects/BEVFusion/bevfusion/bevfusion_head.py#L404-L422) `predict_by_feat`。 + +--- + +## §29–33 Decode：Feature → Metric 座標 + +文件 §29–32（center/dim/rot/height 的 decode）與 §33 的完整數值範例，全部在 bbox coder 的一個 `decode`： +[`utils.py:126-163`](projects/BEVFusion/bevfusion/utils.py#L126-L163) `TransFusionBBoxCoder.decode` +```python +final_preds = heatmap.max(1).indices # label +final_scores = heatmap.max(1).values # score +# §29 center: feature → metric（out_size_factor=8, voxel_size=0.17, +pc_range） +center[:,0,:] = center[:,0,:] * self.out_size_factor * self.voxel_size[0] + self.pc_range[0] +center[:,1,:] = center[:,1,:] * self.out_size_factor * self.voxel_size[1] + self.pc_range[1] +dim = dim.exp() # §30 log dim → 實際尺寸 +height = height - dim[:,2:3,:] * 0.5 # §32 gravity center → bottom center +rot = torch.atan2(rots, rotc) # §31 (sin,cos) → yaw +``` + +--- + +## §34 Postprocess：threshold + post-center-range + Circle NMS + +文件的後處理流程在部署層 `postprocess()`，直接呼叫模型的 coder（`filter=True`）與 cluster/circle NMS，複現 test.py： +[`bevfusion_inference_pipeline.py:145-246`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L145-L246) +```python +# filter=True → coder 內套用 per-class score_threshold + post_center_range +decoded = bbox_coder.decode(heatmap, rot, dim, center, height, vel, filter=True)[0] +boxes3d, scores, labels = apply_cluster_nms( + decoded["bboxes"], decoded["scores"], decoded["labels"], + nms_type=bbox_head.test_cfg.get("nms_type"), + nms_clusters=getattr(bbox_head, "nms_clusters", []), ...) +``` +- per-class `score_threshold` / `post_center_range` 的實際套用： + [`utils.py:176-200`](projects/BEVFusion/bevfusion/utils.py#L176-L200)（`decode` 的 filter 分支）。 +- Circle / cluster NMS：`apply_cluster_nms` 定義於 + [`utils.py`](projects/BEVFusion/bevfusion/utils.py#L1)（import 於 + [`bevfusion_inference_pipeline.py:21`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L21)）。 +- 輸出 detection dict（`bbox_3d / score / label`）： + [`bevfusion_inference_pipeline.py:225-246`](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L225-L246)。 + +--- + +## §35–37 完整對照速查 + +| 文件章節 / 操作 | 檔案:行 | 函式 / 符號 | +|---|---|---| +| §2–3 Voxelization | [inference:85](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L85) / [bevfusion.py:262](projects/BEVFusion/bevfusion/bevfusion.py#L262) | `preprocess` → `pts_voxel_layer` / `voxelize` | +| §4 Voxel encoder | [voxel_encoder:48](projects/BEVFusion/bevfusion/bevfusion_voxel_encoder.py#L48) | `HardSimpleVoxelSinCosEncoder.forward` | +| §5 Sparse encoder | [sparse_encoder:122](projects/BEVFusion/bevfusion/sparse_encoder.py#L122) | `BEVFusionSparseEncoder.forward` | +| §5 sparse 入口(部署) | [onnx:52](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L52) / [bevfusion.py:200](projects/BEVFusion/bevfusion/bevfusion.py#L200) | `BEVFusionSparseWrapper` / `extract_pts_feat` | +| §6 Scatter + Z collapse | [sparse_encoder:154](projects/BEVFusion/bevfusion/sparse_encoder.py#L154) | `sparse_to_dense` + `view` | +| §7–8 SECOND + FPN | [onnx:74](deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py#L74) | `BEVFusionDenseWrapper` → `pts_backbone/pts_neck` | +| §10 Shared conv | [head:269](projects/BEVFusion/bevfusion/bevfusion_head.py#L269) | `shared_conv` | +| §11 Dense heatmap | [head:280](projects/BEVFusion/bevfusion/bevfusion_head.py#L280) | `heatmap_head` + `sigmoid` | +| §12 Local-max | [head:283](projects/BEVFusion/bevfusion/bevfusion_head.py#L283) | `F.max_pool2d` + `heatmap==local_max` | +| §13 Flatten | [head:316](projects/BEVFusion/bevfusion/bevfusion_head.py#L316) | `view` | +| §14 Top-500 + index 反解 | [head:322](projects/BEVFusion/bevfusion/bevfusion_head.py#L322) | `torch.topk` / `//`,`%` | +| §15 Gather query | [head:328](projects/BEVFusion/bevfusion/bevfusion_head.py#L328) | `fusion_feat_flatten.gather` | +| §16 Query pos | [head:340](projects/BEVFusion/bevfusion/bevfusion_head.py#L340) | `bev_pos` + `self_posembed` | +| §17 Class embed | [head:335](projects/BEVFusion/bevfusion/bevfusion_head.py#L335) | `class_encoding` | +| §18–25 Transformer | [head:348](projects/BEVFusion/bevfusion/bevfusion_head.py#L348) / [transformer:26](projects/BEVFusion/bevfusion/transformer.py#L26) | `decoder[i]` / `TransformerDecoderLayer` | +| §26 Prediction heads | [head:353](projects/BEVFusion/bevfusion/bevfusion_head.py#L353) | `prediction_heads[i]` | +| §27 Center += pos | [head:354](projects/BEVFusion/bevfusion/bevfusion_head.py#L354) | `center + query_pos` | +| §28 Score | [head_outputs:25](deployment/projects/bevfusion_l/io/head_outputs.py#L25) | `head_dict_to_detection_outputs` | +| §29–33 Decode | [utils:126](projects/BEVFusion/bevfusion/utils.py#L126) | `TransFusionBBoxCoder.decode` | +| §34 threshold+NMS | [inference:212](deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py#L212) | `decode(filter=True)` + `apply_cluster_nms` | + +--- + +### 一句話總結程式碼結構 + +> **部署層**只做兩件事：graph 外的 `preprocess`(voxelize) 與 `postprocess`(decode+NMS)，中間把模型切成 `Sparse`/`Dense` 兩個 wrapper 匯出。 +> **模型層**才是文件描述的真正運算：`extract_pts_feat`（§4–6）→ `pts_backbone/pts_neck`（§7–8）→ `bbox_head.forward_single`（§9–28）→ `TransFusionBBoxCoder.decode`（§29–33）。 diff --git a/deployment/projects/bevfusion/docs/README.md b/deployment/projects/bevfusion_l/docs/README.md similarity index 54% rename from deployment/projects/bevfusion/docs/README.md rename to deployment/projects/bevfusion_l/docs/README.md index 24eabe8b0..dce348268 100644 --- a/deployment/projects/bevfusion/docs/README.md +++ b/deployment/projects/bevfusion_l/docs/README.md @@ -6,7 +6,9 @@ evaluation). The architecture map is in the parent [`README.md`](../README.md). | # | File | Topic | |---|------|--------| | 25 | [`25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md`](./25_README_AUTOWARE_COORD_CONTRACT_AND_EVAL_ALIGNMENT.md) | `coors` contract alignment with Autoware: why old/new ONNX both evaluate correctly, relation to ROS `x/y/z` and its boundaries | -| 26 | [`26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](./26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) | Why `ScatterND -> SECOND` differs between `original` and split-merge ONNX; why separate tracing needs less shape-plumbing; numerical impact | +| 26 | [`26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md`](./26_README_SCATTERND_TO_SECOND_TRACE_DIFFERENCE.md) | Why `ScatterND -> SECOND` differs between `original` and split-merge ONNX; why separate tracing needs less shape-plumbing; numerical impact — **§2–3 partially corrected by doc 29** | | 28 | [`28_README_BEVFUSION_2_8_DEPLOYMENT.md`](./28_README_BEVFUSION_2_8_DEPLOYMENT.md) | BEVFusion 2.8.x deployment notes | +| 29 | [`29_README_ONNX_NODE_COUNT_ALIGNMENT.md`](./29_README_ONNX_NODE_COUNT_ALIGNMENT.md) | 對齊 split 與 monolithic ONNX 節點數:commit `78b66a70` 的 clean-export 改動、`lidar_bev` `dynamic_axes` 陷阱與修正(524→416≈423)、殘留差異;更正 doc 26 | +| 30 | [`30_README_EVALUATION_PIPELINE_WALKTHROUGH.md`](./30_README_EVALUATION_PIPELINE_WALKTHROUGH.md) | **完整 evaluation 導覽**:CLI→entrypoint→runner→orchestrator→evaluator loop→pipeline(前處理/sparse+dense/後處理)→metrics 的逐檔逐函式呼叫鏈;BEVFusion 各模型部件與 ONNX 各部件在做什麼;split/merged 佈局、TensorRT 執行、T4MetricV2 計分 | -Python entrypoints, configs, and pipelines live in the parent directory (`deployment/projects/bevfusion/`), alongside this `docs/` folder. +Python entrypoints, configs, and pipelines live in the parent directory (`deployment/projects/bevfusion_l/`), alongside this `docs/` folder. diff --git a/deployment/projects/bevfusion_l/entrypoint.py b/deployment/projects/bevfusion_l/entrypoint.py new file mode 100644 index 000000000..cda13717f --- /dev/null +++ b/deployment/projects/bevfusion_l/entrypoint.py @@ -0,0 +1,31 @@ +"""BEVFusion-L deployment entrypoint invoked by the unified CLI.""" + +from __future__ import annotations + +import argparse + +from mmengine.config import Config + +from deployment.config.base import BaseDeploymentConfig +from deployment.execution.backend_executor import BackendExecutor +from deployment.projects.bevfusion_l.config.bevfusion_deployment_config import BEVFusionDeploymentConfig +from deployment.projects.bevfusion_l.evaluation.executor import BEVFusionExecutor +from deployment.projects.bevfusion_l.runner import BEVFusionDeploymentRunner +from deployment.runtime.detection3d_entrypoint import run_detection3d_deployment + + +def _build_executor(config: BaseDeploymentConfig, deploy_cfg: Config) -> BackendExecutor: + """Build the BEVFusion executor, forwarding the spconv ImplicitGemm plugin ``.so`` paths.""" + plugin_libraries = tuple((deploy_cfg.get("tensorrt_config", {}) or {}).get("plugin_libraries", ()) or ()) + return BEVFusionExecutor(components_cfg=config.components_cfg, plugin_libraries=plugin_libraries) + + +def run(args: argparse.Namespace) -> int: + """Run the BEVFusion-L deployment workflow via the shared 3D-detection entrypoint.""" + return run_detection3d_deployment( + args, + pipeline_name="BEVFusion", + config_factory=BEVFusionDeploymentConfig, + executor_factory=_build_executor, + runner_factory=BEVFusionDeploymentRunner, + ) diff --git a/deployment/projects/bevfusion/evaluation/__init__.py b/deployment/projects/bevfusion_l/evaluation/__init__.py similarity index 100% rename from deployment/projects/bevfusion/evaluation/__init__.py rename to deployment/projects/bevfusion_l/evaluation/__init__.py diff --git a/deployment/projects/bevfusion_l/evaluation/executor.py b/deployment/projects/bevfusion_l/evaluation/executor.py new file mode 100644 index 000000000..197a87fe1 --- /dev/null +++ b/deployment/projects/bevfusion_l/evaluation/executor.py @@ -0,0 +1,97 @@ +""" +BEVFusion backend executor. + +Implements the task-specific backend execution primitives (pipeline creation and +input preparation) for BEVFusion, shared by the evaluator and the verification +runner via `~deployment.execution.backend_executor.BackendExecutor`. + +This replaces the OLD ``BEVFusionPipelineFactory`` (the global pipeline registry was removed +in the refactor): pipeline construction uses the reference model on ``self.pytorch_model`` +(set by the runner after export). +""" + +import logging +from typing import Iterable, List, Optional + +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.config.schema import ComponentsConfig +from deployment.execution.point_cloud_backend_executor import PointCloudBackendExecutor +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import ModelSpec +from deployment.projects.bevfusion_l.config.component_layout import has_component, is_split_components +from deployment.projects.bevfusion_l.inference.pytorch_inference_pipeline import BEVFusionPyTorchInferencePipeline +from deployment.projects.bevfusion_l.inference.tensorrt_inference_pipeline import BEVFusionTensorRTInferencePipeline + +logger = logging.getLogger(__name__) + + +class BEVFusionExecutor(PointCloudBackendExecutor): + """Backend execution primitives for BEVFusion (pipeline creation, input prep). + + Args: + components_cfg: Unified components configuration, forwarded to the ONNX/TensorRT + pipelines so they can resolve split (sparse+dense) vs merged full-graph artifacts. + plugin_libraries: Custom TensorRT plugin ``.so`` paths forwarded to the + TensorRT pipeline (e.g. the spconv ImplicitGemm plugin); empty when none is needed. + """ + + def __init__(self, components_cfg: ComponentsConfig, plugin_libraries: Iterable[str] = ()) -> None: + super().__init__() + self._components_cfg = components_cfg + self._plugin_libraries = tuple(plugin_libraries) + + @override + def get_supported_backends(self) -> List[Backend]: + """BEVFusion supports PyTorch and TensorRT only. + + ONNXRuntime cannot run the sparse (spconv) graph: it relies on ``autoware``-domain + custom ops (``ImplicitGemm`` / ``GetIndicePairsImplicitGemm``) that are TensorRT plugins + (``libautoware_tensorrt_plugins.so``), not ORT ops. ONNX *export* still happens (it is the + PyTorch→ONNX→TensorRT bridge); only ONNX *inference* is unsupported here. + """ + return [Backend.PYTORCH, Backend.TENSORRT] + + @override + def get_output_names(self) -> Optional[List[str]]: + """Return the model output names (split→dense outputs; otherwise merged-graph outputs).""" + if is_split_components(self._components_cfg) and not has_component(self._components_cfg, "bevfusion_merged"): + comp = self._components_cfg.get_component("bevfusion_dense") + else: + comp = self._components_cfg.get_component("bevfusion_merged") + return [out.name for out in comp.io.outputs] + + @override + def create_pipeline(self, model_spec: ModelSpec, device: DeviceSpec) -> BaseInferencePipeline: + """Create a BEVFusion inference pipeline for the given backend and device. + + Args: + model_spec: Model specification (backend, device, path). + device: Target device for the pipeline. + + Returns: + BEVFusion pipeline instance (PyTorch, ONNX, or TensorRT). + + Raises: + ValueError: If ``model_spec.backend`` is not a supported backend. + """ + backend = model_spec.backend + self._validate_backend(backend) + + if backend is Backend.PYTORCH: + logger.info("Creating BEVFusion PyTorch pipeline on %s", device) + return BEVFusionPyTorchInferencePipeline(self.pytorch_model, device=device) + + if backend is Backend.TENSORRT: + logger.info("Creating BEVFusion TensorRT pipeline from %s on %s", model_spec.artifact.path, device) + return BEVFusionTensorRTInferencePipeline( + self.pytorch_model, + tensorrt_dir=model_spec.artifact.path, + device=device, + components_cfg=self._components_cfg, + plugin_libraries=self._plugin_libraries, + ) + + raise ValueError(f"Unsupported backend: {backend.value}") diff --git a/deployment/projects/bevfusion/export/__init__.py b/deployment/projects/bevfusion_l/export/__init__.py similarity index 100% rename from deployment/projects/bevfusion/export/__init__.py rename to deployment/projects/bevfusion_l/export/__init__.py diff --git a/deployment/projects/bevfusion_l/export/component_builder.py b/deployment/projects/bevfusion_l/export/component_builder.py new file mode 100644 index 000000000..9df4021f1 --- /dev/null +++ b/deployment/projects/bevfusion_l/export/component_builder.py @@ -0,0 +1,138 @@ +"""BEVFusion-specific component builder. + +Splits a BEVFusion model into ONNX-exportable components for the shared ``OnnxExportPipeline``: + +- **split** (``bevfusion_sparse`` + ``bevfusion_dense``): the sparse encoder and the dense + (SECOND+neck+head) graph as two components. The dense component's tracing input (``lidar_bev``) + is produced by running the sparse encoder on the sample — the same "run the earlier stage to get + the later stage's input" pattern CenterPoint uses for its backbone component. + +The single full-graph ONNX (``bevfusion_merged``) is not exported directly; it is composed from +the split sparse+dense pair as a post-export finalize step (see ``transforms.py`` and +``bevfusion_merge``), so this builder only ever produces the split pair. + +The builder is a pure ``model + export-ready sample -> components`` step. :class:`BEVFusionVoxelSample` +already carries tensors on the model device with ``coors`` in the int32 ``[z, y, x]`` graph-input +layout (see :class:`BEVFusionSampleExtractor`), so the builder never touches device or dtype. The +two export-time globals live elsewhere by design: the SparseConv+BN fold is applied at model load +(``build_bevfusion_model(fuse_spconv_bn=...)``) and ``spconv_do_sort`` is set by the runner before +export — neither is a per-component concern. +""" + +from __future__ import annotations + +import logging +from functools import partial +from typing import List + +import torch + +from deployment.export.pipelines.component_builder import ExportableComponent, ModelComponentBuilder +from deployment.projects.bevfusion_l.config.bevfusion_deployment_config import BEVFusionDeploymentConfig +from deployment.projects.bevfusion_l.config.component_layout import is_split_components +from deployment.projects.bevfusion_l.export.onnx_fuse_implicit_gemm_activation import ( + fuse_autoware_implicit_gemm_trailing_relu, +) +from deployment.projects.bevfusion_l.export.onnx_models.bevfusion_onnx import ( + BEVFusionDenseWrapper, + BEVFusionSparseWrapper, +) +from deployment.projects.bevfusion_l.export.transforms import fix_topk_constant_k +from deployment.projects.bevfusion_l.io.sample_types import BEVFusionVoxelSample + +logger = logging.getLogger(__name__) + + +def _voxel_inputs(sample: BEVFusionVoxelSample) -> tuple: + """The (voxels, coors, num_points) tracing tuple, taken from the export-ready sample as-is.""" + return (sample.voxels, sample.coors, sample.num_points_per_voxel) + + +def _num_proposals(model: torch.nn.Module) -> int: + """Return the head's ``num_proposals`` — the constant K baked into the exported TopK node.""" + head = getattr(model, "bbox_head", None) + if head is None or not hasattr(head, "num_proposals"): + raise ValueError("BEVFusion bbox_head.num_proposals is required for the TopK-constant fix.") + return int(head.num_proposals) + + +def _topk_fix(model: torch.nn.Module): + """The TopK-constant post-export transform, bound to the model's ``num_proposals``.""" + return partial(fix_topk_constant_k, num_proposals=_num_proposals(model)) + + +def _fuse_implicit_gemm_relu(model_proto): + """Post-export transform: fold trailing ReLU into ImplicitGemm ``act_type``.""" + n_relu = fuse_autoware_implicit_gemm_trailing_relu(model_proto) + logger.info("Sparse ONNX postprocess: ImplicitGemm fuse done (trailing Relu=%d).", n_relu) + return model_proto + + +def _run_sparse_encoder(model: torch.nn.Module, sample: BEVFusionVoxelSample) -> torch.Tensor: + """Run the sparse encoder on the sample to get a BEV feature map for tracing the dense graph.""" + with torch.no_grad(): + return BEVFusionSparseWrapper(model).eval()(*_voxel_inputs(sample)) + + +class BEVFusionComponentBuilder(ModelComponentBuilder): + """Build exportable BEVFusion components (the split sparse + dense pair).""" + + def __init__(self, config: BEVFusionDeploymentConfig) -> None: + """Store the deploy config (component layout + ``spconv_fuse_implicit_gemm_relu`` flag).""" + self._config = config + + def build_components( + self, + model: torch.nn.Module, + sample: BEVFusionVoxelSample, + ) -> List[ExportableComponent]: + """Build the ``sparse`` + ``dense`` component pair (the only supported export layout). + + The single full-graph ONNX (``bevfusion_merged``) is composed from this pair afterwards by + the merge finalize hook, so the builder never exports a full graph directly. + """ + if not is_split_components(self._config.components_cfg): + raise ValueError( + "BEVFusion export requires the split sparse+dense layout; the merged full graph is " + "derived from that pair post-export (see bevfusion_merge / transforms.py)." + ) + logger.info("Building BEVFusion split components (sparse + dense)...") + return [self._sparse_component(model, sample), self._dense_component(model, sample)] + + def _sparse_component(self, model: torch.nn.Module, sample: BEVFusionVoxelSample) -> ExportableComponent: + """Sparse encoder: voxels/coors/num_points -> ``lidar_bev`` (optional ImplicitGemm+ReLU fuse).""" + if self._config.spconv_fuse_implicit_gemm_relu: + post_transforms: tuple = (_fuse_implicit_gemm_relu,) + else: + post_transforms = () + logger.info("Sparse ONNX postprocess: ImplicitGemm ReLU fuse disabled by deploy config.") + return self._component( + "bevfusion_sparse", BEVFusionSparseWrapper(model), _voxel_inputs(sample), post_transforms + ) + + def _dense_component(self, model: torch.nn.Module, sample: BEVFusionVoxelSample) -> ExportableComponent: + """Dense graph: ``lidar_bev`` -> detection triple. Traced with a BEV map from the sparse encoder.""" + lidar_bev = _run_sparse_encoder(model, sample) + logger.info("Dense trace input lidar_bev shape: %s", tuple(lidar_bev.shape)) + return self._component("bevfusion_dense", BEVFusionDenseWrapper(model), (lidar_bev,), (_topk_fix(model),)) + + def _component( + self, + name: str, + module: torch.nn.Module, + sample_input: tuple, + post_transforms: tuple, + ) -> ExportableComponent: + """Assemble one ``ExportableComponent``, taking its canonical name from the deploy config. + + Looking the name up here (rather than passing a literal) keeps the exported component name + in lockstep with the deploy config and validates the component exists — the same pattern + CenterPoint's builder uses. + """ + component_cfg = self._config.components_cfg.get_component(name) + return ExportableComponent( + name=component_cfg.name, + module=module, + sample_input=sample_input, + post_transforms=post_transforms, + ) diff --git a/deployment/projects/bevfusion_l/export/onnx_fuse_implicit_gemm_activation.py b/deployment/projects/bevfusion_l/export/onnx_fuse_implicit_gemm_activation.py new file mode 100644 index 000000000..b90c51f7e --- /dev/null +++ b/deployment/projects/bevfusion_l/export/onnx_fuse_implicit_gemm_activation.py @@ -0,0 +1,143 @@ +"""Fuse a post-spconv activation into the ``autoware`` ImplicitGemm plugin. + +TensorRT does not fuse a standard ONNX ``Relu`` with a custom op, so we fold the +pattern ``ImplicitGemm -> Relu`` by hand: + + * set ``act_type = kReLU`` on the ImplicitGemm node, and + * delete the now-redundant standalone ``Relu`` node. + +The public entry point is :func:`fuse_autoware_implicit_gemm_trailing_relu`. +""" + +from __future__ import annotations + +from typing import Dict, List + +import onnx +from onnx import helper + +# ImplicitGemm ``act_type`` values (mirrors the plugin's enum). +_ACT_NONE = 0 +_ACT_RELU = 1 + +_ONNX_DOMAINS = ("", "ai.onnx") + + +def _normalize_attr(name: str) -> str: + """Strip an ONNX type suffix (``_f``, ``_i``, ``_s``, ``_l``) from an attribute name.""" + for suffix in ("_f", "_i", "_s", "_l"): + if name.endswith(suffix) and len(name) > len(suffix): + return name[: -len(suffix)] + return name + + +def _read_int_and_float_attrs(node: onnx.NodeProto) -> Dict[str, object]: + """Return the node's INT/FLOAT attributes keyed by their normalized name.""" + attrs: Dict[str, object] = {} + for attr in node.attribute: + base = _normalize_attr(attr.name) + if attr.type == onnx.AttributeProto.INT: + attrs[base] = int(attr.i) + elif attr.type == onnx.AttributeProto.FLOAT: + attrs[base] = float(attr.f) + return attrs + + +def _set_act_type(node: onnx.NodeProto, act_type: int) -> None: + """Replace the node's ``act_type`` attribute with ``act_type``.""" + kept = [a for a in node.attribute if _normalize_attr(a.name) != "act_type"] + del node.attribute[:] + node.attribute.extend(kept) + node.attribute.append(helper.make_attribute("act_type", int(act_type))) + + +def _rename_tensor(graph: onnx.GraphProto, old: str, new: str) -> None: + """Rewire every reference to tensor ``old`` so it points at ``new`` instead.""" + if old == new: + return + for node in graph.node: + for i, inp in enumerate(node.input): + if inp == old: + node.input[i] = new + for out in graph.output: + if out.name == old: + out.name = new + for value_info in graph.value_info: + if value_info.name == old: + value_info.name = new + + +def _is_onnx_relu(node: onnx.NodeProto) -> bool: + return ( + node.op_type == "Relu" + and node.domain in _ONNX_DOMAINS + and len(node.input) >= 1 + and bool(node.input[0]) + and len(node.output) >= 1 + and bool(node.output[0]) + ) + + +def _is_autoware_implicit_gemm(node: onnx.NodeProto) -> bool: + return node.op_type == "ImplicitGemm" and node.domain == "autoware" + + +def fuse_autoware_implicit_gemm_trailing_relu(model: onnx.ModelProto) -> int: + """Fold each ``ImplicitGemm -> Relu`` pair into a single activated ImplicitGemm. + + For every ONNX ``Relu`` whose input is produced by an ``autoware.ImplicitGemm`` + that feeds nothing else, the ImplicitGemm's ``act_type`` is set to kReLU and the + ``Relu`` node is removed. + + Returns the number of ``Relu`` nodes removed. + """ + graph = model.graph + + # Map each tensor to the node index producing it, and count consumers per tensor. + # Built once up front: the only nodes we remove are the fused Relus, which never + # feed an ImplicitGemm, so removals cannot change any decision made below. + producer_of: Dict[str, int] = {} + consumer_count: Dict[str, int] = {} + for ni, node in enumerate(graph.node): + for out in node.output: + if out: + producer_of[out] = ni + for inp in node.input: + if inp: + consumer_count[inp] = consumer_count.get(inp, 0) + 1 + + remove_idx: set[int] = set() + + for ri, relu in enumerate(graph.node): + if not _is_onnx_relu(relu): + continue + + gemm_out = relu.input[0] + + # The ImplicitGemm output must feed this Relu and nothing else. + if consumer_count.get(gemm_out, 0) != 1: + continue + + producer_i = producer_of.get(gemm_out) + if producer_i is None: + continue + producer = graph.node[producer_i] + if not _is_autoware_implicit_gemm(producer): + continue + + # Only fuse when the ImplicitGemm has no activation yet (or already kReLU). + cur_act = int(_read_int_and_float_attrs(producer).get("act_type", 0) or 0) + if cur_act not in (_ACT_NONE, _ACT_RELU): + continue + + _set_act_type(producer, _ACT_RELU) + _rename_tensor(graph, relu.output[0], gemm_out) + remove_idx.add(ri) + + if not remove_idx: + return 0 + + kept: List[onnx.NodeProto] = [n for i, n in enumerate(graph.node) if i not in remove_idx] + del graph.node[:] + graph.node.extend(kept) + return len(remove_idx) diff --git a/deployment/projects/bevfusion/inference/__init__.py b/deployment/projects/bevfusion_l/export/onnx_models/__init__.py similarity index 100% rename from deployment/projects/bevfusion/inference/__init__.py rename to deployment/projects/bevfusion_l/export/onnx_models/__init__.py diff --git a/deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py b/deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py new file mode 100644 index 000000000..9a5f1b980 --- /dev/null +++ b/deployment/projects/bevfusion_l/export/onnx_models/bevfusion_onnx.py @@ -0,0 +1,83 @@ +"""BEVFusion deploy-only ONNX module wrappers. + +These wrappers adapt BEVFusion submodules to the ONNX export interface (fixed input/output +signatures) and are the ``module`` fed to the shared ``OnnxExportPipeline`` via +``BEVFusionComponentBuilder``. They replicate the containers from the legacy +``projects/BEVFusion/deploy/containers.py`` within the new deployment framework. + +- :class:`BEVFusionSparseWrapper`: LiDAR sparse encoder only (voxels/coors/num_points -> BEV feature map). +- :class:`BEVFusionDenseWrapper`: SECOND + neck + head (+ postprocess) on a BEV feature map. + +The full LiDAR graph (voxels/coors/num_points -> detection triple) is not wrapped here: it is +composed from the exported sparse + dense ONNX by the merge finalize hook (see ``transforms.py``), +so no single full-graph module wrapper is needed. +""" + +from __future__ import annotations + +import torch +import torch.nn as nn + +from deployment.projects.bevfusion_l.io.head_outputs import head_dict_to_detection_outputs +from deployment.projects.bevfusion_l.io.voxel_inputs import graph_input_zyx_to_model_indices_xyz + + +def normalize_sparse_coors_for_autoware(coors: torch.Tensor) -> torch.Tensor: + """Normalize sparse coordinates to the legacy Autoware export contract. + + Graph **inputs** must be ``[z, y, x]`` (no batch). This wrapper flips to + ``[x, y, z]`` and prepends batch — same as ``projects/BEVFusion/deploy/containers.py``. + Voxelization outputs ``[x, y, z]``; convert with ``voxel_indices_xyz_to_graph_input_zyx`` + before tracing or feeding ONNX/TRT. + """ + # Guard that spconv gets int32 indices. Conditional so tracing an already-int32 input (the + # normal case — the graph input is declared int32) emits no redundant no-op Cast in the ONNX. + if coors.dtype != torch.int32: + coors = coors.to(dtype=torch.int32) + if coors.shape[1] == 3: + num_points = coors.shape[0] + coors = graph_input_zyx_to_model_indices_xyz(coors) + batch_coors = torch.zeros(num_points, 1, dtype=torch.int32, device=coors.device) + coors = torch.cat([batch_coors, coors], dim=1).contiguous() + return coors + + +class BEVFusionSparseWrapper(nn.Module): + """LiDAR sparse encoder only: voxels/coors/num_points → BEV feature map.""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.mod = model + + def forward( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> torch.Tensor: + # Keep voxels FP32 for spconv. Conditional so an already-FP32 trace input (the normal + # case — the graph input is declared float32) emits no redundant no-op Cast in the ONNX. + if voxels.dtype != torch.float32: + voxels = voxels.to(dtype=torch.float32) + coors = normalize_sparse_coors_for_autoware(coors) + + return self.mod.extract_pts_feat(voxels, coors, num_points_per_voxel, points=None) + + +class BEVFusionDenseWrapper(nn.Module): + """SECOND + neck + head (+ ONNX postprocess). Input: ``lidar_bev`` [B,C,H,W].""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.mod = model + + def forward(self, lidar_bev: torch.Tensor) -> tuple: + x = lidar_bev + if self.mod.pts_backbone is not None: + x = self.mod.pts_backbone(x) + if self.mod.pts_neck is not None: + x = self.mod.pts_neck(x) + x = self.mod._align_lidar_bev_to_head_grid(x) + outputs = self.mod.bbox_head(x, []) + head_out = outputs[0][0] + return head_dict_to_detection_outputs(head_out) diff --git a/deployment/projects/bevfusion_l/export/sample_extractor.py b/deployment/projects/bevfusion_l/export/sample_extractor.py new file mode 100644 index 000000000..d0fd40ebc --- /dev/null +++ b/deployment/projects/bevfusion_l/export/sample_extractor.py @@ -0,0 +1,54 @@ +"""BEVFusion export sample extractor. + +Produces the typed tracing sample consumed by :class:`BEVFusionComponentBuilder`: it loads a +point-cloud sample and runs BEVFusion voxelization, returning the voxel features, the sparse +coordinates in the ONNX graph-input ``[z, y, x]`` layout, and the per-voxel point counts. +""" + +from __future__ import annotations + +import torch + +from deployment.export.pipelines.sample_extractor import SampleExtractor +from deployment.io.base_data_loader import BaseDataLoader +from deployment.projects.bevfusion_l.io.sample_types import BEVFusionVoxelSample +from deployment.projects.bevfusion_l.io.voxel_inputs import voxel_indices_xyz_to_graph_input_zyx + + +class BEVFusionSampleExtractor(SampleExtractor): + """Extract a voxelized BEVFusion sample for ONNX export tracing.""" + + def extract_sample( + self, + model: torch.nn.Module, + data_loader: BaseDataLoader, + sample_idx: int, + ) -> BEVFusionVoxelSample: + """Load a sample and voxelize it into a typed tracing payload. + + Args: + model: BEVFusion model exposing ``pts_voxel_layer`` (used for voxelization). + data_loader: Loader providing ``load_sample(sample_idx)`` with a ``points`` tensor. + sample_idx: Index of the sample to trace with. + + Returns: + A :class:`BEVFusionVoxelSample` with tensors on the model's device. + """ + sample = data_loader.load_sample(sample_idx) + points = sample["points"] + + device = next(model.parameters()).device + points = points.to(device).float() + + with torch.no_grad(): + ret = model.pts_voxel_layer(points) + if len(ret) == 3: + feats, coords, sizes = ret + else: + feats, coords = ret + sizes = torch.ones(feats.shape[0], device=device) + + coords = coords[:, :].to(dtype=torch.int32) # [M, 3] (x, y, z) from voxel layer + coords = voxel_indices_xyz_to_graph_input_zyx(coords) # ONNX graph input: [z, y, x] + + return BEVFusionVoxelSample(voxels=feats, coors=coords, num_points_per_voxel=sizes) diff --git a/deployment/projects/bevfusion/export/spconv_bn_fusion.py b/deployment/projects/bevfusion_l/export/spconv_bn_fusion.py similarity index 100% rename from deployment/projects/bevfusion/export/spconv_bn_fusion.py rename to deployment/projects/bevfusion_l/export/spconv_bn_fusion.py diff --git a/deployment/projects/bevfusion_l/export/transforms.py b/deployment/projects/bevfusion_l/export/transforms.py new file mode 100644 index 000000000..e747a4e50 --- /dev/null +++ b/deployment/projects/bevfusion_l/export/transforms.py @@ -0,0 +1,197 @@ +"""BEVFusion post-export ONNX graph transforms. + +Pure ``onnx.ModelProto`` graph rewrites applied after ``torch.onnx.export``: + +- :func:`fix_topk_constant_k`: replace the head's dynamic TopK ``K`` with a constant + (``num_proposals``) — TensorRT requires a constant K. Registered as a per-component + ``post_transforms`` on the shared ``OnnxExportPipeline``. +- :func:`merge_split_sparse_dense_onnx`: compose the split ``sparse`` + ``dense`` ONNX into a + single ``merged`` ONNX. Used as the pipeline-level ``finalize`` hook for split+merge exports. + +The ImplicitGemm+ReLU fusion lives in :mod:`onnx_fuse_implicit_gemm_activation` and is registered +directly as a ``post_transforms`` entry by the component builder. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import List + +import numpy as np +import onnx +import onnx_graphsurgeon as gs + +from deployment.projects.bevfusion_l.config.bevfusion_deployment_config import BEVFusionDeploymentConfig +from deployment.projects.bevfusion_l.config.component_layout import has_component + +logger = logging.getLogger(__name__) + + +def fix_topk_constant_k(model: onnx.ModelProto, num_proposals: int) -> onnx.ModelProto: + """Replace the TopK node's ``K`` with a constant ``num_proposals``. + + TensorRT requires TopK's K to be a constant, but ``torch.onnx.export`` may produce a + dynamic K. This rewrites the (single) TopK node in place and fixes up its output shapes. + + Args: + model: Exported ONNX model containing the head's TopK node. + num_proposals: Constant K to bake into the TopK node. + + Returns: + The same ``model`` with the TopK ``K`` constant-folded (returned for transform chaining). + """ + logger.info("Fixing TopK (K=%s) in ONNX graph...", num_proposals) + graph = gs.import_onnx(model) + + topk_nodes = [node for node in graph.nodes if node.op == "TopK"] + if len(topk_nodes) == 0: + logger.warning("No TopK node found; skipping fix") + return model + + if len(topk_nodes) != 1: + logger.warning("Expected 1 TopK node, found %s; fixing the first one", len(topk_nodes)) + + topk = topk_nodes[0] + topk.inputs[1] = gs.Constant("K", values=np.array([num_proposals], dtype=np.int64)) + topk.outputs[0].shape = [1, num_proposals] + topk.outputs[0].dtype = topk.inputs[0].dtype if topk.inputs[0].dtype else np.float32 + topk.outputs[1].shape = [1, num_proposals] + topk.outputs[1].dtype = np.int64 + + graph.cleanup().toposort() + fixed = gs.export_onnx(graph) + logger.info("TopK fixed (K=%s)", num_proposals) + return fixed + + +def merge_split_sparse_dense_onnx( + *, + config: BEVFusionDeploymentConfig, + sparse_onnx_path: Path, + dense_onnx_path: Path, + output_dir_path: Path, + logger: logging.Logger = logger, +) -> Path: + """Merge split ``sparse`` + ``dense`` ONNX into a single ``merged`` ONNX. + + Args: + config: BEVFusion deploy config (supplies the merged component's I/O names and filename). + sparse_onnx_path: Path to the exported sparse-encoder ONNX. + dense_onnx_path: Path to the exported dense ONNX. + output_dir_path: Directory to write the merged ``merged`` ONNX into. + logger: Logger for progress messages. + + Returns: + The path of the written merged ONNX. + + Raises: + KeyError: If the ``bevfusion_merged`` component is not present in the config. + FileNotFoundError: If either split ONNX is missing. + RuntimeError: If ONNX compose utilities are unavailable. + """ + if not has_component(config.components_cfg, "bevfusion_merged"): + raise KeyError( + "bevfusion_merge is enabled but components_cfg has no 'bevfusion_merged'. " + "Ensure merge overlay is applied before export." + ) + merged_cfg = config.components_cfg.get_component("bevfusion_merged") + merged_path = output_dir_path / merged_cfg.onnx_file + + try: + from onnx import compose as onnx_compose + except Exception as e: + raise RuntimeError("ONNX compose utilities unavailable; cannot merge split ONNX.") from e + + if not sparse_onnx_path.exists(): + raise FileNotFoundError(f"Sparse ONNX not found: {sparse_onnx_path}") + if not dense_onnx_path.exists(): + raise FileNotFoundError(f"Dense ONNX not found: {dense_onnx_path}") + + sparse_model = onnx.load(str(sparse_onnx_path)) + dense_model = onnx.load(str(dense_onnx_path)) + + # onnx.compose.merge_models requires identical IR/opset metadata. + target_ir = max(int(sparse_model.ir_version), int(dense_model.ir_version)) + sparse_model.ir_version = target_ir + dense_model.ir_version = target_ir + + sparse_opsets = {imp.domain: int(imp.version) for imp in sparse_model.opset_import} + dense_opsets = {imp.domain: int(imp.version) for imp in dense_model.opset_import} + merged_opsets = dict(sparse_opsets) + for domain, version in dense_opsets.items(): + merged_opsets[domain] = max(version, merged_opsets.get(domain, version)) + merged_opset_ids = [onnx.helper.make_operatorsetid(d, v) for d, v in merged_opsets.items()] + del sparse_model.opset_import[:] + sparse_model.opset_import.extend(merged_opset_ids) + del dense_model.opset_import[:] + dense_model.opset_import.extend(merged_opset_ids) + + sparse_pref = onnx_compose.add_prefix(sparse_model, prefix="sparse/") + dense_pref = onnx_compose.add_prefix(dense_model, prefix="dense/") + + sparse_out_name = config.components_cfg.get_component("bevfusion_sparse").io.outputs[0].name + dense_in_name = config.components_cfg.get_component("bevfusion_dense").io.inputs[0].name + io_map = [(f"sparse/{sparse_out_name}", f"dense/{dense_in_name}")] + + merged_model = onnx_compose.merge_models(sparse_pref, dense_pref, io_map=io_map) + merged_graph = gs.import_onnx(merged_model) + + sparse_inputs = [inp.name for inp in config.components_cfg.get_component("bevfusion_sparse").io.inputs] + dense_outputs = [out.name for out in config.components_cfg.get_component("bevfusion_dense").io.outputs] + if len(merged_graph.inputs) != len(sparse_inputs): + logger.warning( + "Merged ONNX input count mismatch: graph=%d expected=%d", + len(merged_graph.inputs), + len(sparse_inputs), + ) + if len(merged_graph.outputs) != len(dense_outputs): + logger.warning( + "Merged ONNX output count mismatch: graph=%d expected=%d", + len(merged_graph.outputs), + len(dense_outputs), + ) + for i, name in enumerate(sparse_inputs): + if i < len(merged_graph.inputs): + merged_graph.inputs[i].name = name + for i, name in enumerate(dense_outputs): + if i < len(merged_graph.outputs): + merged_graph.outputs[i].name = name + + merged_graph.cleanup().toposort() + final_model = gs.export_onnx(merged_graph) + # onnx.compose.merge_models concatenated both sub-models' (already-unified) opset lists, so the + # merged graph lists each domain twice. Restore the single deduped union computed above so the + # merged ONNX carries the same opset_import a monolithic single-graph export would. + del final_model.opset_import[:] + final_model.opset_import.extend(merged_opset_ids) + onnx.save_model(final_model, str(merged_path)) + logger.info("Merged split ONNX -> %s", merged_path) + return merged_path + + +def bevfusion_merge_finalize( + exported_paths: List[str], + output_dir_path: Path, + config: BEVFusionDeploymentConfig, +) -> None: + """Pipeline finalize hook: merge the split sparse+dense ONNX into a single ``merged`` ONNX. + + Matches :data:`deployment.export.pipelines.onnx_export_pipeline.FinalizeHook`. Resolves the + split ONNX paths from the deploy config under ``output_dir_path`` and delegates to + :func:`merge_split_sparse_dense_onnx`. + + Args: + exported_paths: Per-component ONNX paths already written (unused; paths are resolved + from the deploy config to avoid depending on export order). + output_dir_path: Directory holding the exported ONNX files. + config: BEVFusion deploy config with the split + merged component layout. + """ + sparse_onnx = output_dir_path / config.components_cfg.get_component("bevfusion_sparse").onnx_file + dense_onnx = output_dir_path / config.components_cfg.get_component("bevfusion_dense").onnx_file + merge_split_sparse_dense_onnx( + config=config, + sparse_onnx_path=sparse_onnx, + dense_onnx_path=dense_onnx, + output_dir_path=output_dir_path, + ) diff --git a/deployment/projects/bevfusion/io/__init__.py b/deployment/projects/bevfusion_l/inference/__init__.py similarity index 100% rename from deployment/projects/bevfusion/io/__init__.py rename to deployment/projects/bevfusion_l/inference/__init__.py diff --git a/deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py b/deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py new file mode 100644 index 000000000..b51f9a405 --- /dev/null +++ b/deployment/projects/bevfusion_l/inference/bevfusion_inference_pipeline.py @@ -0,0 +1,284 @@ +"""BEVFusion Inference Pipeline Base Class. + +Provides common preprocessing, postprocessing, and inference logic shared by the PyTorch and +TensorRT backend implementations. ONNXRuntime is not a runtime backend for BEVFusion: the sparse +(spconv) graph needs TensorRT-only ``autoware`` plugins, so ONNX is an export format only. +""" + +from __future__ import annotations + +import logging +import time +from abc import abstractmethod +from typing import Any, Dict, List, Mapping, Tuple, Union + +import torch +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.primitives.device import DeviceSpec +from projects.BEVFusion.bevfusion.utils import apply_cluster_nms + +logger = logging.getLogger(__name__) + + +class BEVFusionInferencePipeline(BaseInferencePipeline): + """Base pipeline for BEVFusion inference. + + Handles voxelization in preprocessing and bbox decoding in postprocessing. + The model (ONNX/TensorRT) takes voxels/coors/num_points_per_voxel and + outputs bbox_pred/score/label_pred directly. + """ + + def __init__( + self, + pytorch_model: torch.nn.Module, + backend_type: Backend, + device: DeviceSpec, + ) -> None: + """Initialize BEVFusion pipeline. + + Args: + pytorch_model: PyTorch model for preprocessing/postprocessing. + backend_type: Deployment backend enum. Required. + device: Target runtime device (DeviceSpec). + + Raises: + ValueError: If class_names not found in pytorch_model.cfg. + """ + cfg = getattr(pytorch_model, "cfg", None) + + class_names = getattr(cfg, "class_names", None) + + if class_names is None: + raise ValueError("class_names not found in pytorch_model.cfg") + + super().__init__( + model=pytorch_model, + backend_type=backend_type, + device=device, + ) + + self.pytorch_model: torch.nn.Module = pytorch_model + self.num_classes: int = len(class_names) + self.class_names: List[str] = class_names + + @override + def preprocess( + self, + points: torch.Tensor, + ) -> Dict[str, torch.Tensor]: + """Voxelize point cloud into voxels/coors/num_points_per_voxel. + + Uses the BEVFusion model's voxelization layer (outside the ONNX graph). + + Args: + points: Point cloud tensor [N, point_features]. + + Returns: + Dict with voxels, coors, num_points_per_voxel. + """ + points_tensor = self.to_device_tensor(points).float() + + with torch.no_grad(): + voxel_output = self.pytorch_model.pts_voxel_layer(points_tensor) + if not (isinstance(voxel_output, (tuple, list)) and len(voxel_output) == 3): + raise NotImplementedError( + "BEVFusion deployment only supports hard voxelization " + "(max_num_points > 0); got a voxel layer output that is not " + "(voxels, coors, num_points_per_voxel)." + ) + voxels, coors, num_points_per_voxel = voxel_output + + preprocessed_dict = { + "voxels": voxels, + "coors": coors, + "num_points_per_voxel": num_points_per_voxel, + } + return preprocessed_dict + + @override + def run_model( + self, + preprocessed_input: Dict[str, torch.Tensor], + ) -> Tuple[List[torch.Tensor], Dict[str, float]]: + """Run the BEVFusion model as two stages and report per-stage latency. + + Mirrors :class:`~deployment.projects.centerpoint.inference.centerpoint_inference_pipeline.CenterPointInferencePipeline`: + the base orchestrates named seams and times each, so every backend shares one + ``sparse`` / ``dense`` breakdown instead of hand-rolling its own. BEVFusion's two + seams line up with the split ONNX/TensorRT graphs: + + - :meth:`run_sparse_encoder`: ``pts_voxel_encoder`` + ``pts_middle_encoder`` (spconv) + -> the dense BEV feature map (the ``bevfusion_sparse`` component). + - :meth:`run_dense`: ``pts_backbone`` + ``pts_neck`` + ``bbox_head`` + scoring + -> ``[bbox_pred, score, label_pred]`` (the ``bevfusion_dense`` component). + + Plain wall-clock timing, exactly like CenterPoint's base orchestration: accurate for + ONNX (``ort.run`` is a blocking call) and reported as reference timings for the native + PyTorch backend. TensorRT overrides this to substitute pure-GPU CUDA-event times, and the + merged single-graph backends override it to report one total (they cannot be split cleanly). + + Args: + preprocessed_input: Dict with voxels, coors, num_points_per_voxel. + + Returns: + Tuple of ([bbox_pred, score, label_pred], {"sparse_ms", "dense_ms"}). + """ + stage_latencies: Dict[str, float] = {} + + start = time.perf_counter() + bev_features = self.run_sparse_encoder( + preprocessed_input["voxels"], + preprocessed_input["coors"], + preprocessed_input["num_points_per_voxel"], + ) + stage_latencies["sparse_ms"] = (time.perf_counter() - start) * 1000 + + start = time.perf_counter() + model_outputs = self.run_dense(bev_features) + stage_latencies["dense_ms"] = (time.perf_counter() - start) * 1000 + + return model_outputs, stage_latencies + + @override + def postprocess( + self, + model_output: List[torch.Tensor], + metadata: Mapping[str, Any], + ) -> List[Dict[str, Union[List[float], float, int]]]: + """Decode bbox_pred/score/label_pred into detection dicts. + + The ONNX graph already bakes in the query scoring that produces the (bbox_pred, score, + label_pred) triple, but the remaining reference-eval selection is not in the graph and is + reproduced here so PyTorch-deploy / TensorRT match test.py: the bbox_coder decode with + ``filter=True`` (per-class ``score_threshold`` + ``post_center_range``) followed by + per-cluster circle NMS. bbox outputs arrive in head-encoded space and are decoded to + metric coordinates: + - bbox_pred: [10, num_proposals] + (center_x_feat, center_y_feat, z_gravity, dim0_log, dim1_log, dim2_log, sin, cos, vx, vy) + - score: [num_proposals] + - label_pred: [num_proposals] + + Args: + model_output: [bbox_pred, score, label_pred] tensors. + metadata: Sample metadata. + + Returns: + List of detection dicts with bbox_3d, score, label. + """ + bbox_pred, score, label_pred = [self.to_device_tensor(o) for o in model_output] + + # Normalize common export/runtime shapes to [10, num_proposals], [num_proposals], [num_proposals]. + if bbox_pred.ndim == 3 and bbox_pred.shape[0] == 1: + bbox_pred = bbox_pred[0] + if bbox_pred.ndim == 2 and bbox_pred.shape[0] != 10 and bbox_pred.shape[1] == 10: + bbox_pred = bbox_pred.transpose(0, 1).contiguous() + if bbox_pred.ndim != 2 or bbox_pred.shape[0] != 10: + logger.warning("Unexpected bbox_pred shape %s; skipping frame.", tuple(bbox_pred.shape)) + return [] + + score = score.reshape(-1) + label_pred = label_pred.reshape(-1) + + # bbox_pred/score/label_pred all carry the same num_proposals by construction + # (head_dict_to_detection_outputs derives them from one head-output dict), so the + # bbox_pred column count is the single source of truth for the proposal count. + num_proposals = bbox_pred.shape[1] + + # Decode via BEVFusion's own bbox_coder to avoid convention drift. + bbox_coder = getattr(self.pytorch_model.bbox_head, "bbox_coder", None) + if bbox_coder is None: + logger.warning("bbox_coder not found on model.bbox_head; skipping frame.") + return [] + + center = bbox_pred[0:2, :num_proposals].unsqueeze(0) + height = bbox_pred[2:3, :num_proposals].unsqueeze(0) + dim = bbox_pred[3:6, :num_proposals].unsqueeze(0) + rot = bbox_pred[6:8, :num_proposals].unsqueeze(0) + vel = bbox_pred[8:10, :num_proposals].unsqueeze(0) + + labels = label_pred[:num_proposals].long() + scores = score[:num_proposals].to(dtype=bbox_pred.dtype) + heatmap = torch.zeros((1, self.num_classes, num_proposals), device=self.torch_device, dtype=bbox_pred.dtype) + valid = (labels >= 0) & (labels < self.num_classes) + if valid.any(): + valid_idx = torch.nonzero(valid, as_tuple=False).reshape(-1) + heatmap[0, labels[valid_idx], valid_idx] = scores[valid_idx] + + # filter=True applies the coder's per-class ``score_threshold`` and ``post_center_range``, + # matching what ``BEVFusionHead.predict_by_feat`` runs during the reference eval (test.py). + decoded = bbox_coder.decode(heatmap, rot, dim, center, height, vel, filter=True)[0] + bbox_head = self.pytorch_model.bbox_head + boxes3d, scores, labels = apply_cluster_nms( + decoded["bboxes"], + decoded["scores"], + decoded["labels"], + nms_type=bbox_head.test_cfg.get("nms_type"), + nms_clusters=getattr(bbox_head, "nms_clusters", []), + box_type_3d=metadata.get("box_type_3d"), + pre_max_size=bbox_head.test_cfg.get("pre_max_size"), + post_max_size=bbox_head.test_cfg.get("post_max_size"), + ) + + results: List[Dict[str, Union[List[float], float, int]]] = [] + for i in range(boxes3d.shape[0]): + bbox = boxes3d[i].detach().cpu().numpy() + # decoded box format: [x, y, z, dx, dy, dz, yaw, vx, vy] + if bbox.shape[0] < 7: + continue + + cx, cy, z = float(bbox[0]), float(bbox[1]), float(bbox[2]) + d0, d1, d2 = float(bbox[3]), float(bbox[4]), float(bbox[5]) + yaw = float(bbox[6]) + vx = float(bbox[7]) if bbox.shape[0] > 7 else 0.0 + vy = float(bbox[8]) if bbox.shape[0] > 8 else 0.0 + + results.append( + { + "bbox_3d": [cx, cy, z, d0, d1, d2, yaw, vx, vy], + "score": float(scores[i].item()), + "label": int(labels[i].item()), + } + ) + + return results + + @abstractmethod + def run_sparse_encoder( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> torch.Tensor: + """Run the sparse branch: voxel encoder + spconv middle encoder -> dense BEV. + + Analogous to CenterPoint's :meth:`run_voxel_encoder`; corresponds to the + ``bevfusion_sparse`` ONNX/TensorRT component. Voxelization already happened in + :meth:`preprocess`, so this consumes the voxel tensors directly. + + Args: + voxels: [M, max_points, C] + coors: [M, 3] (z, y, x) + num_points_per_voxel: [M] + + Returns: + The dense BEV feature map [B, C, H, W] fed to :meth:`run_dense`. + """ + raise NotImplementedError + + @abstractmethod + def run_dense(self, bev_features: torch.Tensor) -> List[torch.Tensor]: + """Run the dense branch: backbone + neck + head + scoring -> detection tensors. + + Analogous to CenterPoint's :meth:`run_backbone_head`; corresponds to the + ``bevfusion_dense`` ONNX/TensorRT component. + + Args: + bev_features: Dense BEV feature map [B, C, H, W] from :meth:`run_sparse_encoder`. + + Returns: + ``[bbox_pred, score, label_pred]`` in the config's declared output order. + """ + raise NotImplementedError diff --git a/deployment/projects/bevfusion_l/inference/pytorch_inference_pipeline.py b/deployment/projects/bevfusion_l/inference/pytorch_inference_pipeline.py new file mode 100644 index 000000000..0d0de9326 --- /dev/null +++ b/deployment/projects/bevfusion_l/inference/pytorch_inference_pipeline.py @@ -0,0 +1,87 @@ +"""BEVFusion PyTorch Pipeline Implementation (sparse + dense seams).""" + +from __future__ import annotations + +import logging +from typing import List + +import torch +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.primitives.device import DeviceSpec +from deployment.projects.bevfusion_l.inference.bevfusion_inference_pipeline import BEVFusionInferencePipeline +from deployment.projects.bevfusion_l.io.head_outputs import head_dict_to_detection_outputs + +logger = logging.getLogger(__name__) + + +class BEVFusionPyTorchInferencePipeline(BEVFusionInferencePipeline): + """PyTorch-based BEVFusion pipeline (sparse + dense seams). + + Runs the full model natively, split into the same ``sparse`` / ``dense`` seams the + ONNX/TensorRT backends use so outputs and the latency breakdown line up across backends. + The base :meth:`run_model` brackets each seam with CUDA syncs and reports ``sparse_ms`` / + ``dense_ms``. + """ + + def __init__(self, pytorch_model: torch.nn.Module, device: DeviceSpec) -> None: + super().__init__(pytorch_model=pytorch_model, backend_type=Backend.PYTORCH, device=device) + logger.info("BEVFusion PyTorch pipeline initialized (sparse/dense seams)") + + @override + def run_sparse_encoder( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> torch.Tensor: + """Sparse branch: voxel encoder (mean pool + sin-cos Fourier) + spconv -> dense BEV.""" + # Model is already loaded in eval mode by ``build_bevfusion_model``; no per-sample eval() here. + model = self.pytorch_model + device = self.torch_device + + voxels = voxels.to(device) + coors = coors.to(device) + num_points_per_voxel = num_points_per_voxel.to(device) + + with torch.no_grad(): + if coors.shape[1] == 3: + num_points = coors.shape[0] + batch_coors = torch.zeros(num_points, 1, device=device, dtype=coors.dtype) + coors = torch.cat([batch_coors, coors], dim=1).contiguous() + + voxel_features = model.pts_voxel_encoder(voxels, num_points_per_voxel, coors) + spatial_features = model.pts_middle_encoder(voxel_features, coors, batch_size=1) + + return spatial_features + + @override + def run_dense(self, bev_features: torch.Tensor) -> List[torch.Tensor]: + """Dense branch: backbone (SECOND) + neck (SECONDFPN) + bbox_head + scoring.""" + model = self.pytorch_model + + with torch.no_grad(): + backbone_out = bev_features + if hasattr(model, "pts_backbone") and model.pts_backbone is not None: + backbone_out = model.pts_backbone(bev_features) + + neck_out = backbone_out + if hasattr(model, "pts_neck") and model.pts_neck is not None: + neck_out = model.pts_neck(backbone_out) + + # Match ``BEVFusion.extract_feat``: head ``bev_pos`` is built for + # ``grid_size // out_size_factor`` (e.g. 180×180) while SECOND/FPN can + # yield full voxel BEV (e.g. 1440×1440). Skipping this pool causes + # ``key`` vs ``key_pos`` length mismatch in the transformer decoder. + align_fn = getattr(model, "_align_lidar_bev_to_head_grid", None) + if callable(align_fn): + neck_out = align_fn(neck_out) + + preds = model.bbox_head(neck_out, []) + preds = preds[0][0] + + # Shared with the ONNX export contract so PyTorch↔ONNX outputs stay bit-identical. + bbox_pred, score, label_pred = head_dict_to_detection_outputs(preds) + + return [bbox_pred, score, label_pred] diff --git a/deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py b/deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py new file mode 100644 index 000000000..d44a6e90f --- /dev/null +++ b/deployment/projects/bevfusion_l/inference/tensorrt_inference_pipeline.py @@ -0,0 +1,230 @@ +"""BEVFusion TensorRT Pipeline Implementation.""" + +from __future__ import annotations + +import logging +import os.path as osp +from typing import Dict, List, Tuple + +import numpy as np +import pycuda.autoinit # noqa: F401 +import tensorrt as trt +import torch +from typing_extensions import override + +from deployment.config.enums import Backend +from deployment.config.schema import ComponentsConfig +from deployment.inference.gpu_resource_mixin import GPUResourceMixin, release_tensorrt_resources +from deployment.inference.tensorrt_runner import list_trt_io_names, load_trt_engine, run_trt_engine +from deployment.primitives.artifacts import resolve_artifact_path +from deployment.primitives.device import DeviceSpec +from deployment.primitives.tensorrt_plugins import load_tensorrt_plugin_libraries +from deployment.projects.bevfusion_l.config.component_layout import has_component, is_split_components +from deployment.projects.bevfusion_l.inference.bevfusion_inference_pipeline import BEVFusionInferencePipeline +from deployment.projects.bevfusion_l.io.voxel_inputs import map_voxel_inputs, voxel_indices_xyz_to_graph_input_zyx + +logger = logging.getLogger(__name__) + + +class BEVFusionTensorRTInferencePipeline(GPUResourceMixin, BEVFusionInferencePipeline): + """TensorRT-based BEVFusion pipeline (one loaded engine per deploy-config component). + + Engines and contexts are held in ``self._engines`` / ``self._contexts`` keyed by component + name (the same pattern CenterPoint uses), so the layout only decides *which* components are + loaded, not how they are stored: + + - Split layout: ``bevfusion_sparse`` + ``bevfusion_dense``, each CUDA-timed via the + ``run_sparse_encoder`` / ``run_dense`` seams (``sparse_ms`` / ``dense_ms``). + - Merged layout: a single ``bevfusion_merged`` full-graph engine that cannot be split, so + it reports one ``model_ms`` GPU total. + """ + + def __init__( + self, + pytorch_model: torch.nn.Module, + tensorrt_dir: str, + device: DeviceSpec, + components_cfg: ComponentsConfig, + plugin_libraries: Tuple[str, ...] = (), + ) -> None: + super().__init__(pytorch_model=pytorch_model, backend_type=Backend.TENSORRT, device=device) + + self.tensorrt_dir = tensorrt_dir + self._components_cfg = components_cfg + self._plugin_libraries = plugin_libraries + self._trt_logger = trt.Logger(trt.Logger.WARNING) + + # Prefer the merged full-graph engine when the split+merge export produced one on disk; + # otherwise run the split sparse+dense pair. + split_layout = is_split_components(components_cfg) + merged_available = ( + split_layout + and has_component(components_cfg, "bevfusion_merged") + and osp.exists( + resolve_artifact_path( + base_dir=tensorrt_dir, + components_cfg=components_cfg, + component_name="bevfusion_merged", + file_key="engine_file", + ) + ) + ) + self._split = split_layout and not merged_available + + # Engine/context per component name (like CenterPoint); the loaded keys follow the layout. + self._engines: Dict[str, trt.ICudaEngine] = {} + self._contexts: Dict[str, trt.IExecutionContext] = {} + + # Per-stage pure-GPU times (ms), filled by each seam while its CUDA stream is still alive + # and read back in run_model (mirrors the CenterPoint TensorRT pipeline). + self._gpu_stage_ms: Dict[str, float] = {} + + self._load_tensorrt_engines() + logger.info("BEVFusion TensorRT pipeline initialized from: %s (split=%s)", tensorrt_dir, self._split) + + def _load_tensorrt_engines(self) -> None: + """Load one engine/context per component for the active layout into the name-keyed dicts.""" + load_tensorrt_plugin_libraries(self._plugin_libraries) + trt.init_libnvinfer_plugins(self._trt_logger, "") + runtime = trt.Runtime(self._trt_logger) + + component_names = ["bevfusion_sparse", "bevfusion_dense"] if self._split else ["bevfusion_merged"] + for component_name in component_names: + engine_path = resolve_artifact_path( + base_dir=self.tensorrt_dir, + components_cfg=self._components_cfg, + component_name=component_name, + file_key="engine_file", + ) + if not osp.exists(engine_path): + raise FileNotFoundError(f"TensorRT engine not found for {component_name}: {engine_path}") + engine, context = load_trt_engine(runtime, engine_path, component_name=component_name) + self._engines[component_name] = engine + self._contexts[component_name] = context + logger.info("Loaded TensorRT engine: %s (%s)", component_name, engine_path) + + def _engine_context(self, component_name: str) -> Tuple[trt.ICudaEngine, trt.IExecutionContext]: + """Return the loaded (engine, context) for a component, or fail loud if it is absent.""" + engine = self._engines.get(component_name) + context = self._contexts.get(component_name) + if engine is None or context is None: + raise RuntimeError(f"TensorRT engine/context for {component_name!r} is not loaded (layout mismatch).") + return engine, context + + def _trt_infer_voxel_inputs( + self, + engine: trt.ICudaEngine, + context: trt.IExecutionContext, + voxels_np: np.ndarray, + coors_np: np.ndarray, + num_points_np: np.ndarray, + ) -> Tuple[Dict[str, np.ndarray], float]: + """Assemble the multi-input voxel map (sparse/merged engines) and run the engine.""" + input_names, output_names = list_trt_io_names(engine) + input_map = map_voxel_inputs(input_names, voxels=voxels_np, coors=coors_np, num_points=num_points_np) + return run_trt_engine(engine, context, input_map, output_names) + + def _prepare_voxel_inputs( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Convert the voxel tensors to the numpy dtypes/axis order the engine expects.""" + voxels_np = self.to_numpy(voxels, dtype=np.float32) + coors_np = self.to_numpy(voxel_indices_xyz_to_graph_input_zyx(coors), dtype=np.int32) + num_points_np = self.to_numpy(num_points_per_voxel, dtype=np.int32) + # Match ``extract_pts_feat``: mean-pool must not divide by zero (NaN BEV → dense NaN). + num_points_np = np.maximum(num_points_np, 1) + return voxels_np, coors_np, num_points_np + + @override + def run_model( + self, + preprocessed_input: Dict[str, torch.Tensor], + ) -> Tuple[List[torch.Tensor], Dict[str, float]]: + """Split: pure-GPU ``sparse_ms`` / ``dense_ms`` from the two seams. Merged: one ``model_ms``. + + Overrides the base wall-clock orchestration to report CUDA-event GPU times (matching the + CenterPoint TensorRT pipeline). The merged full-graph engine is a single ``execute`` that + cannot be split into sparse/dense, so it reports one GPU total under ``model_ms``. + """ + if self._split: + bev_features = self.run_sparse_encoder( + preprocessed_input["voxels"], + preprocessed_input["coors"], + preprocessed_input["num_points_per_voxel"], + ) + outputs = self.run_dense(bev_features) + return outputs, {"sparse_ms": self._gpu_stage_ms["sparse_ms"], "dense_ms": self._gpu_stage_ms["dense_ms"]} + + outputs, gpu_ms = self._run_merged( + preprocessed_input["voxels"], + preprocessed_input["coors"], + preprocessed_input["num_points_per_voxel"], + ) + return outputs, {"model_ms": gpu_ms} + + @override + def run_sparse_encoder( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> torch.Tensor: + """Sparse (spconv) engine: voxels/coors/num_points -> ``lidar_bev`` (split layout only).""" + engine, context = self._engine_context("bevfusion_sparse") + voxels_np, coors_np, num_points_np = self._prepare_voxel_inputs(voxels, coors, num_points_per_voxel) + + sparse_out, gpu_ms = self._trt_infer_voxel_inputs(engine, context, voxels_np, coors_np, num_points_np) + self._gpu_stage_ms["sparse_ms"] = gpu_ms + + if len(sparse_out) != 1: + raise RuntimeError(f"Sparse engine: expected 1 output, got {list(sparse_out.keys())}") + bev_name = next(iter(sparse_out)) + expected = [o.name for o in self._components_cfg.get_component("bevfusion_sparse").io.outputs] + if expected and bev_name not in expected: + logger.warning( + "[trt-split] sparse engine output tensor is %r but deploy_cfg bevfusion_sparse.io.outputs " + "names=%s — check ONNX export / TRT binding names.", + bev_name, + expected, + ) + bev_arr = np.ascontiguousarray(sparse_out[bev_name].astype(np.float32)) + return torch.from_numpy(bev_arr).to(self.torch_device) + + @override + def run_dense(self, bev_features: torch.Tensor) -> List[torch.Tensor]: + """Dense engine: ``lidar_bev`` -> detection tensors (split layout only).""" + engine, context = self._engine_context("bevfusion_dense") + dense_cfg = self._components_cfg.get_component("bevfusion_dense") + + bev_arr = self.to_numpy(bev_features, dtype=np.float32) + # The dense graph has a single input (``lidar_bev``), so bind by the engine's only input name. + input_names, output_names = list_trt_io_names(engine) + dense_out, gpu_ms = run_trt_engine(engine, context, {input_names[0]: bev_arr}, output_names) + self._gpu_stage_ms["dense_ms"] = gpu_ms + + expected_output_names = [out.name for out in dense_cfg.io.outputs] + ordered_names = self.order_outputs_by_config(list(dense_out.keys()), expected_output_names, strict=False) + return [torch.from_numpy(dense_out[name]).to(self.torch_device) for name in ordered_names] + + def _run_merged( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> Tuple[List[torch.Tensor], float]: + """Run the single full-graph engine and return (detection tensors, pure-GPU ms).""" + engine, context = self._engine_context("bevfusion_merged") + voxels_np, coors_np, num_points_np = self._prepare_voxel_inputs(voxels, coors, num_points_per_voxel) + + output_arrays, gpu_ms = self._trt_infer_voxel_inputs(engine, context, voxels_np, coors_np, num_points_np) + expected_output_names = [out.name for out in self._components_cfg.get_component("bevfusion_merged").io.outputs] + ordered_names = self.order_outputs_by_config(list(output_arrays.keys()), expected_output_names, strict=False) + tensors = [torch.from_numpy(output_arrays[name]).to(self.torch_device) for name in ordered_names] + return tensors, gpu_ms + + def _release_gpu_resources(self) -> None: + """Release every loaded engine/context (uniform across split and merged layouts).""" + release_tensorrt_resources(engines=self._engines, contexts=self._contexts) diff --git a/deployment/projects/bevfusion_l/io/__init__.py b/deployment/projects/bevfusion_l/io/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/projects/bevfusion_l/io/head_outputs.py b/deployment/projects/bevfusion_l/io/head_outputs.py new file mode 100644 index 000000000..0de61c14a --- /dev/null +++ b/deployment/projects/bevfusion_l/io/head_outputs.py @@ -0,0 +1,35 @@ +"""BEVFusion detection-head output contract. + +The transformation from the detection head's raw output dict to the +``(bbox_pred, score, label)`` triple is the output contract that the ONNX graph bakes in. +The PyTorch reference pipeline must produce the *identical* triple for PyTorch↔ONNX parity +to be meaningful, so both call this single function rather than each keeping a copy. +""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F + + +def head_dict_to_detection_outputs(outputs: dict) -> tuple: + """Turn the detection-head output dict into the ``(bbox_pred, score, label)`` outputs. + + Args: + outputs: Detection-head output dict with ``heatmap``, ``query_labels``, + ``query_heatmap_score``, and the ``center``/``height``/``dim``/``rot``/``vel`` regressions. + + Returns: + Tuple ``(bbox_pred [10, num_proposals], score [num_proposals], label [num_proposals])``. + """ + score = outputs["heatmap"].sigmoid() + one_hot = F.one_hot(outputs["query_labels"], num_classes=score.size(1)).permute(0, 2, 1) + score = score * outputs["query_heatmap_score"] * one_hot + score = score[0].max(dim=0)[0] + + bbox_pred = torch.cat( + [outputs["center"][0], outputs["height"][0], outputs["dim"][0], outputs["rot"][0], outputs["vel"][0]], + dim=0, + ) + + return bbox_pred, score, outputs["query_labels"][0] diff --git a/deployment/projects/bevfusion_l/io/model_loader.py b/deployment/projects/bevfusion_l/io/model_loader.py new file mode 100644 index 000000000..51169c659 --- /dev/null +++ b/deployment/projects/bevfusion_l/io/model_loader.py @@ -0,0 +1,82 @@ +"""BEVFusion model loading utilities for deployment.""" + +from __future__ import annotations + +import logging + +import torch +from mmengine.config import Config + +# Imported for their side effect: registering BEVFusion and SparseConvolution modules into the +# MMDet3D registries so ``MODELS.build`` can resolve them during export. +import projects.BEVFusion.bevfusion # noqa: F401 +import projects.SparseConvolution # noqa: F401 +from deployment.io.mmdet3d_model import build_mmdet3d_model +from deployment.primitives.device import DeviceSpec +from deployment.projects.bevfusion_l.export.spconv_bn_fusion import fuse_spconv_bn_in_encoder + +logger = logging.getLogger(__name__) + + +def _require_lidar_only_bevfusion(model: torch.nn.Module) -> None: + """Assert the loaded checkpoint is a LiDAR-only BEVFusion model. + + The ``bevfusion_l`` bundle only deploys the LiDAR path (voxels -> sparse encoder -> dense head); + it has no camera/fusion export. A camera (``bevfusion_c``) or fusion (``bevfusion_cl``) + checkpoint would trace a graph this bundle cannot serve, so fail loud once here at load with a + clear message rather than deep inside ONNX export. ``pts_middle_encoder`` is the sparse encoder + every export path and the PyTorch backend depend on, so its absence is also caught here. + """ + if getattr(model, "fusion_layer", None) is not None: + raise RuntimeError( + "bevfusion_l deploys LiDAR-only BEVFusion, but the loaded checkpoint has a fusion_layer. " + "Use a LiDAR-only checkpoint (a camera/fusion model needs a dedicated bevfusion_c / " + "bevfusion_cl bundle)." + ) + if getattr(model, "img_backbone", None) is not None: + raise RuntimeError( + "bevfusion_l deploys LiDAR-only BEVFusion, but the loaded checkpoint has an img_backbone. " + "Use a LiDAR-only checkpoint (a camera/fusion model needs a dedicated bevfusion_c / " + "bevfusion_cl bundle)." + ) + if getattr(model, "pts_middle_encoder", None) is None: + raise RuntimeError( + "bevfusion_l requires a sparse pts_middle_encoder (LiDAR BEVFusion), but the loaded " + "checkpoint has none." + ) + + +def build_bevfusion_model( + model_cfg: Config, + checkpoint_path: str, + device: DeviceSpec, + *, + fuse_spconv_bn: bool = False, +) -> torch.nn.Module: + """Build a BEVFusion model from config and load checkpoint weights. + + Args: + model_cfg: MMEngine model configuration. + checkpoint_path: Path to .pth checkpoint file. + device: Target device. + fuse_spconv_bn: If True, fuse each ``SparseConvolution`` + ``BatchNorm1d`` pair in + ``pts_middle_encoder`` after ``load_checkpoint`` (eval-mode Conv-BN fold, a graph + optimization for the sparse ONNX export). + + Returns: + Loaded and eval-mode BEVFusion model. + + Raises: + RuntimeError: If the checkpoint is not a LiDAR-only BEVFusion model (see + :func:`_require_lidar_only_bevfusion`). + """ + model = build_mmdet3d_model(model_cfg, checkpoint_path, device) + _require_lidar_only_bevfusion(model) + + if fuse_spconv_bn: + encoder = getattr(model, "pts_middle_encoder", None) + if encoder is not None: + count = fuse_spconv_bn_in_encoder(encoder) + logger.info("Fused %d SparseConv-BN pair(s) in pts_middle_encoder", count) + + return model diff --git a/deployment/projects/bevfusion_l/io/sample_types.py b/deployment/projects/bevfusion_l/io/sample_types.py new file mode 100644 index 000000000..6a1e21992 --- /dev/null +++ b/deployment/projects/bevfusion_l/io/sample_types.py @@ -0,0 +1,31 @@ +"""BEVFusion typed export/tracing sample. + +Mirrors CenterPoint's ``io/sample_types.py``: the typed payload produced by the sample extractor +and consumed by the component builder lives in ``io`` (not in ``export``), so both projects keep +their typed samples in the same place. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + + +@dataclass(frozen=True) +class BEVFusionVoxelSample: + """Voxelized, export-ready BEVFusion tracing sample. + + Produced by :class:`~deployment.projects.bevfusion_l.export.sample_extractor.BEVFusionSampleExtractor`. + All tensors are already on the model's device and in the exported graph's expected dtype/layout, + so consumers (the component builder) use them directly and never re-handle device or dtype. + + Attributes: + voxels: Voxel features ``[M, ...]`` (float32), on the model device. + coors: Sparse coordinates ``[M, 3]`` in graph-input ``[z, y, x]`` layout (int32), on the model device. + num_points_per_voxel: Per-voxel point counts ``[M]``, on the model device. + """ + + voxels: torch.Tensor + coors: torch.Tensor + num_points_per_voxel: torch.Tensor diff --git a/deployment/projects/bevfusion_l/io/voxel_inputs.py b/deployment/projects/bevfusion_l/io/voxel_inputs.py new file mode 100644 index 000000000..58acb45b2 --- /dev/null +++ b/deployment/projects/bevfusion_l/io/voxel_inputs.py @@ -0,0 +1,72 @@ +"""BEVFusion voxel input contract for deploy / ONNX / TensorRT. + +Everything about how voxel data enters the exported graph lives here: + +- The **input names** (``voxels`` / ``coors`` / ``num_points_per_voxel``) declared by the deploy + config and baked into the ONNX / TensorRT graphs, and :func:`map_voxel_inputs` to bind the three + arrays to a session/engine's declared inputs. +- The **coordinate layout** of ``coors``: voxelization (``pts_voxel_layer``) returns indices as + ``[x, y, z]``; the legacy Autoware-compatible ONNX expects graph inputs as ``[z, y, x]`` (no + batch). Inside the exported wrapper the indices are flipped back to ``[x, y, z]`` and a batch + column is prepended before ``pts_middle_encoder`` (``sparse_shape`` is ``[H, W, D]``). PyTorch + evaluation uses ``[batch, x, y, z]`` directly and does not use these flips. +""" + +from __future__ import annotations + +from typing import Dict, Sequence, TypeVar + +import torch + +# Canonical sparse / merged-graph voxel input names. These are declared by the deploy config +# (``components.*.io.inputs``) and baked into the exported ONNX / TensorRT graphs, so they are +# the authoritative names to feed by — no substring guessing. +VOXELS_INPUT = "voxels" +COORS_INPUT = "coors" +NUM_POINTS_INPUT = "num_points_per_voxel" + +_T = TypeVar("_T") + + +def map_voxel_inputs(input_names: Sequence[str], *, voxels: _T, coors: _T, num_points: _T) -> Dict[str, _T]: + """Bind the voxel/coors/num-points arrays to a model's declared input names. + + Args: + input_names: Input tensor names reported by the ONNX session / TensorRT engine. + voxels, coors, num_points: The three input arrays to feed. + + Returns: + A ``{input_name: array}`` feed dict, one entry per name in ``input_names``. + + Raises: + RuntimeError: If an input name is not one of the canonical voxel input names — surfaces + an export/config name mismatch loudly instead of silently dropping an input. + """ + by_name: Dict[str, _T] = {VOXELS_INPUT: voxels, COORS_INPUT: coors, NUM_POINTS_INPUT: num_points} + feed: Dict[str, _T] = {} + for name in input_names: + if name not in by_name: + raise RuntimeError(f"Unexpected model input {name!r}; expected one of {list(by_name)}") + feed[name] = by_name[name] + return feed + + +def _flip_last_axis(coors: torch.Tensor) -> torch.Tensor: + """Reverse the axis order of ``[M, 3]`` indices, validating the shape. + + The two directional helpers below are the *same* reversal (a flip is its own inverse); the + distinct names document the intended direction at each call site. + """ + if coors.ndim != 2 or coors.shape[1] != 3: + raise ValueError(f"Expected [M, 3] coors, got shape {tuple(coors.shape)}") + return coors.flip(dims=[-1]).contiguous() + + +def voxel_indices_xyz_to_graph_input_zyx(coors: torch.Tensor) -> torch.Tensor: + """``[M, 3]`` voxel indices ``[x, y, z]`` → graph input ``[z, y, x]``.""" + return _flip_last_axis(coors) + + +def graph_input_zyx_to_model_indices_xyz(coors: torch.Tensor) -> torch.Tensor: + """``[M, 3]`` graph input ``[z, y, x]`` → model indices ``[x, y, z]`` (wrapper flip).""" + return _flip_last_axis(coors) diff --git a/deployment/projects/bevfusion_l/runner.py b/deployment/projects/bevfusion_l/runner.py new file mode 100644 index 000000000..12dbe7dd8 --- /dev/null +++ b/deployment/projects/bevfusion_l/runner.py @@ -0,0 +1,103 @@ +"""BEVFusion-specific deployment runner.""" + +from __future__ import annotations + +import logging +from typing import Optional + +import torch +from mmengine.config import Config +from typing_extensions import override + +from deployment.evaluation.detection_3d_evaluator import Detection3DEvaluator +from deployment.execution.backend_executor import BackendExecutor +from deployment.export.pipelines.onnx_export_pipeline import OnnxExportPipeline +from deployment.export.pipelines.tensorrt_export_pipeline import TensorRTExportPipeline +from deployment.io.base_data_loader import BaseDataLoader +from deployment.projects.bevfusion_l.config.bevfusion_deployment_config import BEVFusionDeploymentConfig +from deployment.projects.bevfusion_l.export.component_builder import BEVFusionComponentBuilder +from deployment.projects.bevfusion_l.export.sample_extractor import BEVFusionSampleExtractor +from deployment.projects.bevfusion_l.export.transforms import bevfusion_merge_finalize +from deployment.projects.bevfusion_l.io.model_loader import build_bevfusion_model +from deployment.runtime.runner import BaseDeploymentRunner +from projects.SparseConvolution.sparse_functional import set_do_sort + +logger = logging.getLogger(__name__) + + +class BEVFusionDeploymentRunner(BaseDeploymentRunner): + """BEVFusion deployment runner. + + Implements project-specific model loading (LiDAR-only, on CUDA, with the optional SparseConv+BN + fold and the ``spconv_do_sort`` export global) and wires BEVFusion's ``BEVFusionSampleExtractor`` + + ``BEVFusionComponentBuilder`` (plus the split→merge ``finalize`` hook when + ``config.merge_bevfusion`` is set) into the shared ``OnnxExportPipeline``, reusing the + project-agnostic orchestration in ``BaseDeploymentRunner`` and the shared + ``TensorRTExportPipeline``. + + BEVFusion-only deploy-config flags (``fuse_spconv_bn``, ``spconv_do_sort``, + ``spconv_fuse_implicit_gemm_relu``, ``merge_bevfusion``) are typed attributes on + ``BEVFusionDeploymentConfig``. + """ + + def __init__( + self, + data_loader: BaseDataLoader, + evaluator: Detection3DEvaluator, + executor: BackendExecutor, + config: BEVFusionDeploymentConfig, + model_cfg: Config, + onnx_pipeline: Optional[OnnxExportPipeline] = None, + tensorrt_pipeline: Optional[TensorRTExportPipeline] = None, + ) -> None: + # The exported ONNX layout (split sparse+dense, optionally merged into one full graph) is + # driven entirely by the deploy config's ``components``; there is no per-run module selection. + + # Construct the pipelines BEFORE super().__init__, because the base runner forwards them + # straight to the ExportOrchestrator (there is no post-init slot). + if onnx_pipeline is None: + onnx_pipeline = OnnxExportPipeline( + sample_extractor=BEVFusionSampleExtractor(), + component_builder=BEVFusionComponentBuilder(config=config), + finalize=bevfusion_merge_finalize if config.merge_bevfusion else None, + ) + + super().__init__( + data_loader=data_loader, + evaluator=evaluator, + executor=executor, + config=config, + model_cfg=model_cfg, + onnx_pipeline=onnx_pipeline, + tensorrt_pipeline=tensorrt_pipeline, + ) + + @override + def load_pytorch_model(self, checkpoint_path: str) -> torch.nn.Module: + """Load the BEVFusion model onto the CUDA device for export. + + The base runner forwards the returned model to ``executor.set_pytorch_model`` after + export, so PyTorch/ONNX/TensorRT evaluation all reuse this reference. + """ + cuda_device = self.config.device_config.cuda + if cuda_device is None: + raise RuntimeError( + "BEVFusion requires a CUDA device for sparse convolution. Set devices.cuda in deploy config." + ) + + # ``spconv_do_sort`` is a process-global read by GetIndicePairsImplicitGemm at ONNX symbolic + # export and in the spconv forward path. Set it here — once, before any export or inference — + # so the exported sparse graph and PyTorch inference agree. It is deploy-time config, not a + # per-component concern, so it lives on the runner rather than in the component builder. + set_do_sort(self.config.spconv_do_sort) + logger.info( + "spconv_do_sort=%s (baked into GetIndicePairsImplicitGemm.do_sort_i at ONNX export)", + self.config.spconv_do_sort, + ) + + return build_bevfusion_model( + model_cfg=self.model_cfg, + checkpoint_path=checkpoint_path, + device=cuda_device, + fuse_spconv_bn=self.config.fuse_spconv_bn, + ) diff --git a/deployment/projects/centerpoint/README.md b/deployment/projects/centerpoint/README.md index 30cda2c16..b20137b91 100644 --- a/deployment/projects/centerpoint/README.md +++ b/deployment/projects/centerpoint/README.md @@ -10,7 +10,6 @@ From the repository root: python -m deployment.cli.main centerpoint \ deployment/projects/centerpoint/config/deploy_config.py \ \ - --rot-y-axis-reference \ [--log-level INFO] ``` @@ -19,14 +18,13 @@ Example: ```bash python -m deployment.cli.main centerpoint \ deployment/projects/centerpoint/config/deploy_config.py \ - projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py \ - --rot-y-axis-reference + projects/CenterPoint/configs/t4dataset/Centerpoint/second_secfpn_8xb16_121m_j6gen2_base_amp_t4metric_v2.py ``` ## What is project-specific here - Multi-component export with `pts_voxel_encoder` and `pts_backbone_neck_head` -- CenterPoint-specific CLI flag `--rot-y-axis-reference` +- The `rot_y_axis_reference` export option, set in the deploy config (see `CenterPointDeploymentConfig`) - CenterPoint evaluator, loaders, export pipelines, and backend inference pipelines ## Config file @@ -53,14 +51,12 @@ CenterPoint classes are: | Path | Role | | --- | --- | | `__init__.py` | Registers the `centerpoint` `ProjectAdapter` | -| `entrypoint.py` | Builds config, loader, evaluator, runner, and export context | -| `cli.py` | Project-specific CLI flags (`--rot-y-axis-reference`) | +| `entrypoint.py` | Builds config, loader, evaluator, and runner | | `runner.py` | `CenterPointDeploymentRunner` | -| `config/` | Deploy config | +| `config/` | Deploy config and `CenterPointDeploymentConfig` (typed deploy-config keys, e.g. `rot_y_axis_reference`) | | `io/` | Data loading and model loading helpers | | `evaluation/` | `CenterPointEvaluator` and `CenterPointExecutor` | | `inference/` | PyTorch, ONNX, and TensorRT inference pipelines | -| `contexts.py` | `CenterPointExportContext` | | `export/` | CenterPoint export orchestration (builder, sample extractor, `onnx_models/`) | ## Shared docs diff --git a/deployment/projects/centerpoint/__init__.py b/deployment/projects/centerpoint/__init__.py index 7428b5e6b..ea1e659b6 100644 --- a/deployment/projects/centerpoint/__init__.py +++ b/deployment/projects/centerpoint/__init__.py @@ -6,15 +6,14 @@ from __future__ import annotations -from deployment.projects.centerpoint.cli import add_args from deployment.projects.centerpoint.entrypoint import run from deployment.projects.registry import ProjectAdapter, project_registry +# Options that shape the exported graph (e.g. ``rot_y_axis_reference``) live in the deploy config +# (see ``CenterPointDeploymentConfig``) so they are versioned with the artifact, not passed on the CLI. project_registry.register( ProjectAdapter( name="centerpoint", - add_args=add_args, run=run, - required_components=("pts_voxel_encoder", "pts_backbone_neck_head"), ) ) diff --git a/deployment/projects/centerpoint/cli.py b/deployment/projects/centerpoint/cli.py deleted file mode 100644 index cc040e0d9..000000000 --- a/deployment/projects/centerpoint/cli.py +++ /dev/null @@ -1,14 +0,0 @@ -"""CenterPoint CLI extensions.""" - -from __future__ import annotations - -import argparse - - -def add_args(parser: argparse.ArgumentParser) -> None: - """Register CenterPoint-specific CLI flags onto a project subparser.""" - parser.add_argument( - "--rot-y-axis-reference", - action="store_true", - help="Convert rotation to y-axis clockwise reference (CenterPoint ONNX-compatible format)", - ) diff --git a/deployment/projects/centerpoint/config/centerpoint_deployment_config.py b/deployment/projects/centerpoint/config/centerpoint_deployment_config.py new file mode 100644 index 000000000..79f3688c7 --- /dev/null +++ b/deployment/projects/centerpoint/config/centerpoint_deployment_config.py @@ -0,0 +1,41 @@ +"""CenterPoint-specific deployment config. + +Extends :class:`~deployment.config.base.BaseDeploymentConfig` to model the CenterPoint-only +deploy-config keys as typed attributes, so the entrypoint and export pipeline never reach +back into the raw MMEngine ``Config``. This is the typed home for the keys the generic +sections intentionally do not model. +""" + +from __future__ import annotations + +from mmengine.config import Config + +from deployment.config.base import BaseDeploymentConfig + + +class CenterPointDeploymentConfig(BaseDeploymentConfig): + """Deployment config for CenterPoint. + + Adds typed attributes for the CenterPoint-only deploy-config keys: + + - ``rot_y_axis_reference``: output rotation as ``sin(y), cos(x)`` relative to the y-axis in the + exported head, matching the ONNX-compatible output format expected by the runtime + (default ``False``). + """ + + #: Components CenterPoint always splits into for multi-file ONNX/TensorRT export. + _REQUIRED_COMPONENTS = ("pts_voxel_encoder", "pts_backbone_neck_head") + + def __init__(self, deploy_cfg: Config) -> None: + super().__init__(deploy_cfg) + self.rot_y_axis_reference: bool = bool(deploy_cfg.get("rot_y_axis_reference", False)) + self._validate_components() + + def _validate_components(self) -> None: + """Fail early if the deploy config is missing a required CenterPoint component. + + Validated here (rather than via the project registry) so both CenterPoint and BEVFusion + check their component layout the same way — at config construction time. + """ + for component_name in self._REQUIRED_COMPONENTS: + self.components_cfg.get_component(component_name) diff --git a/deployment/projects/centerpoint/config/deploy_config.py b/deployment/projects/centerpoint/config/deploy_config.py index 4cddcf83c..fcb9e4556 100644 --- a/deployment/projects/centerpoint/config/deploy_config.py +++ b/deployment/projects/centerpoint/config/deploy_config.py @@ -63,6 +63,13 @@ sample_idx=0, ) +# Output box rotation as sin(y), cos(x) relative to the y-axis in the exported head, matching the +# ONNX-compatible output format expected by the runtime. Shapes the exported graph, so it lives in +# the config (versioned with the artifact) rather than as a CLI flag. +# - True : y-axis-referenced rotation output +# - False : keep the training-time rotation encoding (default) +rot_y_axis_reference = False + # ONNX Export Settings (shared across all components). onnx_config = dict( opset_version=17, @@ -188,7 +195,7 @@ # especially when FP16 is enabled. # ============================================================================ verification = dict( - enabled=False, + enabled=True, # TODO(vividf): double check the tolerance value tolerance=1, num_verify_samples=1, diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py b/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py deleted file mode 100644 index bed1f4fd6..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_small.py +++ /dev/null @@ -1,166 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - ConvNeXt Small Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "work_dirs/centerpoint-convnext/epoch_5_downsample_conv_first.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir="work_dirs/centerpoint-convnext/small/fp16-downsample-conv-first", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# -# ConvNeXt Small uses BackwardPillarFeatureNet with 10 input channels: -# base (5) + cluster_center (3) + voxel_center (2) = 10 -# Grid size: [1216, 1216, 1] -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 10], - opt_shape=[20000, 32, 10], - max_shape=[64000, 32, 10], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1216, 1216], - opt_shape=[1, 32, 1216, 1216], - max_shape=[1, 32, 1216, 1216], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=20, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=100, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=True, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py b/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py deleted file mode 100644 index 811e137c8..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_convnext_standard.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - ConvNeXt Standard Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "work_dirs/centerpoint-convnext/standard/epoch_30_standard.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="onnx", - work_dir="work_dirs/centerpoint-convnext/standard", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[64000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=100, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py b/deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py deleted file mode 100644 index 1fa465fcc..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_resnet.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - ResNet34 Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "work_dirs/centerpoint_resnet34_exp3.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="none", - work_dir="work_dirs/centerpoint_fp16_resnet_deployment_exp3", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[64000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=100, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py b/deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py deleted file mode 100644 index 162dc42d8..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_resnet_base.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - ResNet34 Backbone (Base) -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "models/2_5/base/centerpoint_resnet34_base_2_5_epoch49.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir="work_dirs/centerpoint_fp16_resnet_deployment_base", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 10], - opt_shape=[20000, 32, 10], - max_shape=[96000, 32, 10], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=-1, - verbose=True, - backends=dict( - pytorch=dict( - enabled=False, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second.py deleted file mode 100644 index c59e7bcc9..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_second.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - SECOND Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "work_dirs/centerpoint_2_5.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="none", - work_dir="work_dirs/centerpoint_fp16_second_deployment_2_5", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[64000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=100, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py deleted file mode 100644 index 995b30b2d..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_5.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - SECOND Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "models/2_5/experiment_j6_gen2/second/epoch_30.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir="models/2_5/experiment_j6_gen2/second/fp16-deployment", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[64000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=100, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py deleted file mode 100644 index a0d577a3f..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_second_2_6.py +++ /dev/null @@ -1,160 +0,0 @@ -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "vivid/bench_comparison/centerpoint_2_6/epoch_29.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# Single literal for deployment output root (used before `export` exists). -_DEPLOY_WORK_DIR = "work_dirs/centerpoint_2_6_fp16" -_WORK_DIR = _DEPLOY_WORK_DIR.rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir=_DEPLOY_WORK_DIR, - onnx_path=_ONNX_DIR, - sample_idx=1, -) - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[96000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=17, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=500, - num_warmup=2, - verbose=True, - backends=dict( - pytorch=dict( - enabled=False, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py b/deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py deleted file mode 100644 index 3e9475dfd..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_second_base.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - SECOND Backbone (Base) -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "models/2_5/base/centerpoint_second_base_2_5_epoch49.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir="work_dirs/centerpoint_fp16_second_deployment_2_5_base", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 10], - opt_shape=[20000, 32, 10], - max_shape=[96000, 32, 10], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=100, - verbose=True, - backends=dict( - pytorch=dict( - enabled=False, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py b/deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py deleted file mode 100644 index 149c30151..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_vov57.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - SECOND Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "models/2_5/experiment_j6_gen2/vov57-v2-downsample/epoch_30.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir="work_dirs/centerpoint-vov57-v2-downsample/fp16-deployment", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[96000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=-1, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py b/deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py deleted file mode 100644 index 6f4dceace..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp16_vov99.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP16 Deployment Configuration - SECOND Backbone -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "models/2_5/experiment_j6_gen2/vov_epoch_30.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="both", - work_dir="work_dirs/centerpoint-vov99/fp16-deployment", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[96000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 1020, 1020], - opt_shape=[1, 32, 1020, 1020], - max_shape=[1, 32, 1020, 1020], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp16", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=-1, - verbose=True, - backends=dict( - pytorch=dict( - enabled=True, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/config/deploy_config_fp32.py b/deployment/projects/centerpoint/config/deploy_config_fp32.py deleted file mode 100644 index 1df466c44..000000000 --- a/deployment/projects/centerpoint/config/deploy_config_fp32.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -CenterPoint FP32 Deployment Configuration -""" - -# ============================================================================ -# Checkpoint Path -# ============================================================================ -checkpoint_path = "vivid_model/best_checkpoint.pth" - -deploy_log_path = "deployment.log" - -# ============================================================================ -# Device settings -# ============================================================================ -devices = dict( - cpu="cpu", - cuda="cuda:0", -) - -# ============================================================================ -# Export Configuration -# ============================================================================ -export = dict( - mode="none", - work_dir="work_dirs/centerpoint_deployment_fp32", - onnx_path=None, - sample_idx=1, -) - -# Derived artifact directories -_WORK_DIR = str(export["work_dir"]).rstrip("/") -_ONNX_DIR = f"{_WORK_DIR}/onnx" -_TENSORRT_DIR = f"{_WORK_DIR}/tensorrt" - -# ============================================================================ -# Unified Component Configuration -# ============================================================================ -components = dict( - pts_voxel_encoder=dict( - onnx_file="pts_voxel_encoder.onnx", - engine_file="pts_voxel_encoder.engine", - io=dict( - inputs=[ - dict(name="input_features", dtype="float32"), - ], - outputs=[ - dict(name="pillar_features", dtype="float32"), - ], - dynamic_axes={ - "input_features": {0: "num_voxels", 1: "num_max_points"}, - "pillar_features": {0: "num_voxels"}, - }, - ), - tensorrt_profile=dict( - input_features=dict( - min_shape=[1000, 32, 11], - opt_shape=[20000, 32, 11], - max_shape=[64000, 32, 11], - ), - ), - ), - pts_backbone_neck_head=dict( - onnx_file="pts_backbone_neck_head.onnx", - engine_file="pts_backbone_neck_head.engine", - io=dict( - inputs=[ - dict(name="spatial_features", dtype="float32"), - ], - outputs=[ - dict(name="heatmap", dtype="float32"), - dict(name="reg", dtype="float32"), - dict(name="height", dtype="float32"), - dict(name="dim", dtype="float32"), - dict(name="rot", dtype="float32"), - dict(name="vel", dtype="float32"), - ], - dynamic_axes={ - "spatial_features": {0: "batch_size", 2: "height", 3: "width"}, - "heatmap": {0: "batch_size", 2: "height", 3: "width"}, - "reg": {0: "batch_size", 2: "height", 3: "width"}, - "height": {0: "batch_size", 2: "height", 3: "width"}, - "dim": {0: "batch_size", 2: "height", 3: "width"}, - "rot": {0: "batch_size", 2: "height", 3: "width"}, - "vel": {0: "batch_size", 2: "height", 3: "width"}, - }, - ), - tensorrt_profile=dict( - spatial_features=dict( - min_shape=[1, 32, 760, 760], - opt_shape=[1, 32, 760, 760], - max_shape=[1, 32, 760, 760], - ), - ), - ), -) - -# ============================================================================ -# ONNX Export Settings -# ============================================================================ -onnx_config = dict( - opset_version=16, - do_constant_folding=True, - export_params=True, - keep_initializers_as_inputs=False, - simplify=False, -) - -# ============================================================================ -# TensorRT Build Settings -# ============================================================================ -tensorrt_config = dict( - precision_policy="fp32_tf32", - max_workspace_size=4 << 30, -) - -# ============================================================================ -# Evaluation Configuration -# ============================================================================ -evaluation = dict( - enabled=True, - num_samples=-1, - verbose=True, - backends=dict( - pytorch=dict( - enabled=False, - device=devices["cuda"], - ), - onnx=dict( - enabled=False, - device=devices["cuda"], - model_dir=_ONNX_DIR, - ), - tensorrt=dict( - enabled=True, - device=devices["cuda"], - engine_dir=_TENSORRT_DIR, - ), - ), -) - -# ============================================================================ -# Verification Configuration -# ============================================================================ -verification = dict( - enabled=False, - tolerance=1e-1, - num_verify_samples=1, - devices=devices, - scenarios=dict( - both=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - onnx=[ - dict(ref_backend="pytorch", ref_device="cpu", test_backend="onnx", test_device="cpu"), - ], - trt=[ - dict(ref_backend="onnx", ref_device="cuda", test_backend="tensorrt", test_device="cuda"), - ], - none=[], - ), -) diff --git a/deployment/projects/centerpoint/contexts.py b/deployment/projects/centerpoint/contexts.py deleted file mode 100644 index 5f7f35737..000000000 --- a/deployment/projects/centerpoint/contexts.py +++ /dev/null @@ -1,21 +0,0 @@ -"""CenterPoint-specific export context.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from deployment.export.contexts import ExportContext - - -@dataclass(frozen=True) -class CenterPointExportContext(ExportContext): - """ - CenterPoint-specific export context. - - Attributes: - rot_y_axis_reference: Whether to use y-axis rotation reference for - ONNX-compatible output format. This affects - how rotation and dimensions are encoded. - """ - - rot_y_axis_reference: bool = False diff --git a/deployment/projects/centerpoint/entrypoint.py b/deployment/projects/centerpoint/entrypoint.py index 4f0a23c59..220023bb4 100644 --- a/deployment/projects/centerpoint/entrypoint.py +++ b/deployment/projects/centerpoint/entrypoint.py @@ -6,68 +6,25 @@ from mmengine.config import Config -from deployment.cli.args import add_deployment_file_logging, setup_logging from deployment.config.base import BaseDeploymentConfig -from deployment.metrics.detection_3d_metrics import extract_t4metric_v2_config -from deployment.projects.centerpoint.contexts import CenterPointExportContext -from deployment.projects.centerpoint.evaluation.evaluator import CenterPointEvaluator +from deployment.execution.backend_executor import BackendExecutor +from deployment.projects.centerpoint.config.centerpoint_deployment_config import CenterPointDeploymentConfig from deployment.projects.centerpoint.evaluation.executor import CenterPointExecutor -from deployment.projects.centerpoint.io.data_loader import CenterPointDataLoader from deployment.projects.centerpoint.runner import CenterPointDeploymentRunner -from deployment.projects.registry import project_registry +from deployment.runtime.detection3d_entrypoint import run_detection3d_deployment -def run(args: argparse.Namespace) -> int: - """Run the CenterPoint deployment workflow for the unified CLI. - - Args: - args: Parsed command-line arguments containing deploy_cfg and model_cfg paths. - - Returns: - Exit code (0 for success). - """ - logger = setup_logging(args.log_level) - - deploy_cfg = Config.fromfile(args.deploy_cfg) - model_cfg = Config.fromfile(args.model_cfg) - config = BaseDeploymentConfig(deploy_cfg) - - log_file = config.resolved_deploy_log_file - if log_file: - add_deployment_file_logging(log_file) - logger.info("Deployment log file: %s", log_file) - - project_registry.validate_required_components("centerpoint", config.components_cfg) - - logger.info("=" * 80) - logger.info("CenterPoint Deployment Pipeline") - logger.info("=" * 80) +def _build_executor(config: BaseDeploymentConfig, deploy_cfg: Config) -> BackendExecutor: + """Build the CenterPoint executor (no custom TensorRT plugins needed).""" + return CenterPointExecutor(components_cfg=config.components_cfg) - data_loader = CenterPointDataLoader( - model_cfg=model_cfg, - ) - logger.info("Loaded %s samples", data_loader.num_samples) - - metrics_config = extract_t4metric_v2_config(model_cfg) - # One executor instance, shared by the evaluator (evaluate/verify) and the runner - # (which hands it the loaded reference model after export). - executor = CenterPointExecutor(components_cfg=config.components_cfg) - - evaluator = CenterPointEvaluator( - model_cfg=model_cfg, - metrics_config=metrics_config, - executor=executor, - ) - - runner = CenterPointDeploymentRunner( - data_loader=data_loader, - evaluator=evaluator, - executor=executor, - config=config, - model_cfg=model_cfg, +def run(args: argparse.Namespace) -> int: + """Run the CenterPoint deployment workflow via the shared 3D-detection entrypoint.""" + return run_detection3d_deployment( + args, + pipeline_name="CenterPoint", + config_factory=CenterPointDeploymentConfig, + executor_factory=_build_executor, + runner_factory=CenterPointDeploymentRunner, ) - - context = CenterPointExportContext(rot_y_axis_reference=bool(getattr(args, "rot_y_axis_reference", False))) - runner.run(context=context) - return 0 diff --git a/deployment/projects/centerpoint/evaluation/evaluator.py b/deployment/projects/centerpoint/evaluation/evaluator.py deleted file mode 100644 index 3920dda5e..000000000 --- a/deployment/projects/centerpoint/evaluation/evaluator.py +++ /dev/null @@ -1,53 +0,0 @@ -"""CenterPoint evaluator for deployment. - -Thin subclass of ``Detection3DEvaluator``: only ``print_results`` (the flat latency-breakdown -layout) is CenterPoint-specific; the metrics hooks (parse/accumulate/build/summarize) are shared -with the base (see ``deployment.evaluation.detection3d_evaluator``). -""" - -import logging - -from typing_extensions import override - -from deployment.evaluation.base_evaluator import EvalResultDict -from deployment.evaluation.detection3d_evaluator import Detection3DEvaluator - -logger = logging.getLogger(__name__) - - -class CenterPointEvaluator(Detection3DEvaluator): - """Evaluator for CenterPoint 3D detection deployment.""" - - @override - def print_results(self, results: EvalResultDict) -> None: - """Log the metrics report, latency statistics, and stage-wise breakdown.""" - metrics_report = self.metrics_interface.format_metrics_report() - for line in metrics_report.rstrip().split("\n"): - logger.info(line) - - if "latency" not in results: - raise ValueError( - "Latency statistics not found in results. Ensure that evaluation has been run with latency tracking." - ) - self._log_latency_stats(results) - - if "latency_breakdown" in results: - breakdown_dict = results["latency_breakdown"].to_dict() - if breakdown_dict: - logger.info("") - logger.info("Stage-wise Latency Breakdown:") - top_level_stages = {"preprocessing_ms", "model_ms", "postprocessing_ms"} - for stage, stats_dict in breakdown_dict.items(): - stage_name = stage.replace("_ms", "").replace("_", " ").title() - output_format = ( - " %-18s: %.2f ± %.2f ms" if stage in top_level_stages else " %-16s: %.2f ± %.2f ms" - ) - logger.info( - output_format, - stage_name, - stats_dict["mean_ms"], - stats_dict["std_ms"], - ) - - logger.info("") - logger.info("Total Samples: %s", results["num_samples"]) diff --git a/deployment/projects/centerpoint/evaluation/executor.py b/deployment/projects/centerpoint/evaluation/executor.py index fecade771..bc50f6204 100644 --- a/deployment/projects/centerpoint/evaluation/executor.py +++ b/deployment/projects/centerpoint/evaluation/executor.py @@ -1,29 +1,83 @@ -"""CenterPoint backend executor. +""" +CenterPoint backend executor. -Thin subclass of ``PointDetectionExecutor``: declares the CenterPoint pipeline classes and -the head output-name lookup. Pipeline creation and ``(points, metainfo)`` input prep are shared -with the base (see ``deployment.evaluation.point_detection_executor``). +Implements the task-specific backend execution primitives (pipeline creation and +input preparation) for CenterPoint, shared by the evaluator and the verification +runner via `~deployment.execution.backend_executor.BackendExecutor`. """ +import logging from typing import List, Optional from typing_extensions import override -from deployment.evaluation.point_detection_executor import PointDetectionExecutor +from deployment.config.enums import Backend +from deployment.config.schema import ComponentsConfig +from deployment.execution.point_cloud_backend_executor import PointCloudBackendExecutor +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import ModelSpec from deployment.projects.centerpoint.inference.onnx_inference_pipeline import CenterPointONNXInferencePipeline from deployment.projects.centerpoint.inference.pytorch_inference_pipeline import CenterPointPyTorchInferencePipeline from deployment.projects.centerpoint.inference.tensorrt_inference_pipeline import CenterPointTensorRTInferencePipeline +logger = logging.getLogger(__name__) + + +class CenterPointExecutor(PointCloudBackendExecutor): + """Backend execution primitives for CenterPoint (pipeline creation, input prep). -class CenterPointExecutor(PointDetectionExecutor): - """Backend execution primitives for CenterPoint (pipeline creation, input prep).""" + Args: + components_cfg: Unified components configuration, forwarded to the pipeline + registry when constructing backend pipelines. + """ - task_name = "CenterPoint" - pytorch_pipeline_cls = CenterPointPyTorchInferencePipeline - onnx_pipeline_cls = CenterPointONNXInferencePipeline - tensorrt_pipeline_cls = CenterPointTensorRTInferencePipeline + def __init__(self, components_cfg: ComponentsConfig) -> None: + super().__init__() + self._components_cfg = components_cfg @override def get_output_names(self) -> Optional[List[str]]: """Return the head output names from the components config for verification logging.""" return [out.name for out in self._components_cfg.get_component("pts_backbone_neck_head").io.outputs] + + @override + def create_pipeline(self, model_spec: ModelSpec, device: DeviceSpec) -> BaseInferencePipeline: + """Create a CenterPoint inference pipeline for the given backend and device. + + Args: + model_spec: Model specification (backend, device, path). + device: Target device for the pipeline. + + Returns: + CenterPoint pipeline instance (PyTorch, ONNX, or TensorRT). + + Raises: + ValueError: If ``model_spec.backend`` is not a supported backend. + """ + backend = model_spec.backend + self._validate_backend(backend) + + if backend is Backend.PYTORCH: + logger.info("Creating CenterPoint PyTorch pipeline on %s", device) + return CenterPointPyTorchInferencePipeline(self.pytorch_model, device=device) + + if backend is Backend.ONNX: + logger.info("Creating CenterPoint ONNX pipeline from %s on %s", model_spec.artifact.path, device) + return CenterPointONNXInferencePipeline( + self.pytorch_model, + onnx_dir=model_spec.artifact.path, + device=device, + components_cfg=self._components_cfg, + ) + + if backend is Backend.TENSORRT: + logger.info("Creating CenterPoint TensorRT pipeline from %s on %s", model_spec.artifact.path, device) + return CenterPointTensorRTInferencePipeline( + self.pytorch_model, + tensorrt_dir=model_spec.artifact.path, + device=device, + components_cfg=self._components_cfg, + ) + + raise ValueError(f"Unsupported backend: {backend.value}") diff --git a/deployment/projects/centerpoint/export/component_builder.py b/deployment/projects/centerpoint/export/component_builder.py index 9ac055f75..0d9b68757 100644 --- a/deployment/projects/centerpoint/export/component_builder.py +++ b/deployment/projects/centerpoint/export/component_builder.py @@ -10,8 +10,8 @@ import torch -from deployment.config.schema import ComponentsConfig from deployment.export.pipelines.component_builder import ExportableComponent, ModelComponentBuilder +from deployment.projects.centerpoint.config.centerpoint_deployment_config import CenterPointDeploymentConfig from deployment.projects.centerpoint.export.onnx_models.centerpoint_onnx import CenterPointHeadONNX from deployment.projects.centerpoint.io.sample_types import CenterPointFeatureSample, compute_batch_size @@ -28,14 +28,15 @@ class CenterPointComponentBuilder(ModelComponentBuilder): def __init__( self, - components_cfg: ComponentsConfig, + config: CenterPointDeploymentConfig, ) -> None: """Initialize CenterPoint component builder. Args: - components_cfg: Component config used to resolve export names. + config: CenterPoint deploy config supplying the component layout (mirrors + :class:`BEVFusionComponentBuilder`, which also takes its project deploy config). """ - self._components_cfg = components_cfg + self._config = config def build_components( self, @@ -53,13 +54,13 @@ def build_components( """ logger.info("Extracting CenterPoint components for export...") - voxel_component = self._create_voxel_encoder_component(model, sample) - backbone_component = self._create_backbone_component(model, sample) + voxel_component = self._build_voxel_encoder_component(model, sample) + backbone_component = self._build_backbone_component(model, sample) logger.info("Extracted 2 components: pts_voxel_encoder, pts_backbone_neck_head") return [voxel_component, backbone_component] - def _create_voxel_encoder_component( + def _build_voxel_encoder_component( self, model: torch.nn.Module, sample: CenterPointFeatureSample, @@ -73,14 +74,14 @@ def _create_voxel_encoder_component( Returns: Exportable voxel encoder component. """ - component_cfg = self._components_cfg.get_component("pts_voxel_encoder") + component_cfg = self._config.components_cfg.get_component("pts_voxel_encoder") return ExportableComponent( name=component_cfg.name, module=model.pts_voxel_encoder, sample_input=sample.input_features, ) - def _create_backbone_component( + def _build_backbone_component( self, model: torch.nn.Module, sample: CenterPointFeatureSample, @@ -95,9 +96,9 @@ def _create_backbone_component( Exportable backbone/neck/head component. """ backbone_input = self._prepare_backbone_input(model, sample) - backbone_module = self._create_backbone_module(model) + backbone_module = self._build_backbone_module(model) - component_cfg = self._components_cfg.get_component("pts_backbone_neck_head") + component_cfg = self._config.components_cfg.get_component("pts_backbone_neck_head") return ExportableComponent( name=component_cfg.name, module=backbone_module, @@ -125,7 +126,7 @@ def _prepare_backbone_input( spatial_features = model.pts_middle_encoder(voxel_features, coors, batch_size) return spatial_features - def _create_backbone_module(self, model: torch.nn.Module) -> torch.nn.Module: + def _build_backbone_module(self, model: torch.nn.Module) -> torch.nn.Module: """Wrap pts_backbone, pts_neck, and pts_bbox_head into one ONNX module. Args: diff --git a/deployment/projects/centerpoint/inference/centerpoint_inference_pipeline.py b/deployment/projects/centerpoint/inference/centerpoint_inference_pipeline.py index 71de484c7..7b654828a 100644 --- a/deployment/projects/centerpoint/inference/centerpoint_inference_pipeline.py +++ b/deployment/projects/centerpoint/inference/centerpoint_inference_pipeline.py @@ -10,9 +10,8 @@ import logging import time from abc import abstractmethod -from typing import Dict, List, Sequence, Tuple, Union +from typing import Any, Dict, List, Mapping, Tuple, Union -import numpy as np import torch from mmdet3d.structures import Det3DDataSample, LiDARInstance3DBoxes from typing_extensions import override @@ -36,8 +35,6 @@ class CenterPointInferencePipeline(BaseInferencePipeline): pytorch_model: Reference PyTorch model for preprocessing/postprocessing. num_classes: Number of detection classes. class_names: List of class names. - point_cloud_range: Point cloud range [x_min, y_min, z_min, x_max, y_max, z_max]. - voxel_size: Voxel size [vx, vy, vz]. """ def __init__( @@ -56,18 +53,11 @@ def __init__( Raises: ValueError: If class_names not found in pytorch_model.cfg. """ - cfg = pytorch_model.cfg - - class_names = cfg.class_names - point_cloud_range = cfg.point_cloud_range - voxel_size = cfg.voxel_size + cfg = getattr(pytorch_model, "cfg", None) + class_names = getattr(cfg, "class_names", None) if class_names is None: raise ValueError("class_names not found in pytorch_model.cfg") - if point_cloud_range is None: - raise ValueError("point_cloud_range not found in pytorch_model.cfg") - if voxel_size is None: - raise ValueError("voxel_size not found in pytorch_model.cfg") super().__init__( model=pytorch_model, @@ -75,40 +65,10 @@ def __init__( device=device, ) - self.class_names: List[str] = class_names - self.point_cloud_range: List[float] = point_cloud_range - self.voxel_size: List[float] = voxel_size self.pytorch_model: torch.nn.Module = pytorch_model + self.class_names: List[str] = class_names self._rot_y_axis_reference: bool = pytorch_model.pts_bbox_head.rot_y_axis_reference - def to_device_tensor(self, data: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: - """Convert data to tensor on the pipeline's device. - - Args: - data: Input data (torch.Tensor or np.ndarray). - - Returns: - Tensor on pipeline torch device. - """ - if isinstance(data, np.ndarray): - data = torch.from_numpy(data) - return data.to(self.torch_device) - - def to_numpy(self, data: torch.Tensor, dtype: np.dtype = np.float32) -> np.ndarray: - """Convert tensor to contiguous numpy array. - - Args: - data: Input tensor. - dtype: Target numpy dtype. - - Returns: - Contiguous numpy array. - """ - arr = data.cpu().numpy().astype(dtype) - if not arr.flags["C_CONTIGUOUS"]: - arr = np.ascontiguousarray(arr) - return arr - @staticmethod def squeeze_voxel_features(voxel_features: torch.Tensor) -> torch.Tensor: """Collapse the singleton channel of the voxel-encoder output ``[N, 1, F] -> [N, F]``. @@ -121,29 +81,11 @@ def squeeze_voxel_features(voxel_features: torch.Tensor) -> torch.Tensor: raise RuntimeError(f"Expected voxel encoder output [N, 1, F], got shape {tuple(voxel_features.shape)}.") return voxel_features.squeeze(1) - @staticmethod - def order_head_outputs(actual_names: Sequence[str], expected_names: Sequence[str]) -> List[str]: - """Validate backbone-head output names and return them in the configured order. - - ONNX/TensorRT may report outputs in arbitrary order, but CenterPoint postprocess - depends on the exact head order from the component config. This checks for any - missing/extra outputs and returns ``expected_names`` (the config order). - """ - expected_set, actual_set = set(expected_names), set(actual_names) - missing = expected_set - actual_set - extra = actual_set - expected_set - if missing or extra: - raise ValueError( - f"Backbone-head output mismatch: missing={sorted(missing)}, extra={sorted(extra)}; " - f"expected={sorted(expected_set)}, got={sorted(actual_set)}." - ) - return list(expected_names) - @override def preprocess( self, points: torch.Tensor, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, object]]: + ) -> Dict[str, torch.Tensor]: """Preprocess point cloud data for inference. Performs voxelization and feature extraction using the data_preprocessor @@ -153,8 +95,7 @@ def preprocess( points: Point cloud tensor of shape [N, point_features]. Returns: - Tuple of (preprocessed_dict, metadata_dict). - preprocessed_dict contains: input_features, voxels, num_points, coors. + Dict with input_features, voxels, num_points, coors. """ points_tensor = self.to_device_tensor(points) @@ -180,16 +121,14 @@ def preprocess( "coors": coors, } - # Second tuple element: preprocess_metadata for BaseInferencePipeline.infer() - # (merged with caller metadata, then passed to postprocess). Empty here. - return preprocessed_dict, {} + return preprocessed_dict - def process_middle_encoder( + def run_middle_encoder( self, voxel_features: torch.Tensor, coors: torch.Tensor, ) -> torch.Tensor: - """Process voxel features through middle encoder (scatter to BEV). + """Run voxel features through the middle encoder (scatter to BEV). This step runs on PyTorch regardless of backend because it involves sparse-to-dense conversion that's not easily exportable to ONNX. @@ -231,39 +170,39 @@ def run_model( stage_latencies["voxel_encoder_ms"] = (time.perf_counter() - start) * 1000 start = time.perf_counter() - spatial_features = self.process_middle_encoder(voxel_features, preprocessed_input["coors"]) + spatial_features = self.run_middle_encoder(voxel_features, preprocessed_input["coors"]) stage_latencies["middle_encoder_ms"] = (time.perf_counter() - start) * 1000 start = time.perf_counter() - head_outputs = self.run_backbone_head(spatial_features) + model_outputs = self.run_backbone_head(spatial_features) stage_latencies["backbone_head_ms"] = (time.perf_counter() - start) * 1000 - return head_outputs, stage_latencies + return model_outputs, stage_latencies @override def postprocess( self, - head_outputs: List[torch.Tensor], - sample_meta: Dict[str, object], + model_output: List[torch.Tensor], + metadata: Mapping[str, Any], ) -> List[Dict[str, Union[List[float], float, int]]]: """Postprocess head outputs to detection results. Args: - head_outputs: List of 6 tensors [heatmap, reg, height, dim, rot, vel]. - sample_meta: Sample metadata dict. + model_output: List of 6 tensors [heatmap, reg, height, dim, rot, vel]. + metadata: Sample metadata dict. Returns: List of detection dicts with keys: bbox_3d, score, label. Raises: - ValueError: If head_outputs doesn't contain exactly 6 tensors. + ValueError: If model_output doesn't contain exactly 6 tensors. """ - head_outputs = [self.to_device_tensor(out) for out in head_outputs] + model_output = [self.to_device_tensor(out) for out in model_output] - if len(head_outputs) != 6: - raise ValueError(f"Expected 6 head outputs, got {len(head_outputs)}") + if len(model_output) != 6: + raise ValueError(f"Expected 6 head outputs, got {len(model_output)}") - heatmap, reg, height, dim, rot, vel = head_outputs + heatmap, reg, height, dim, rot, vel = model_output # Apply rotation axis correction to mirror the head's export-time convention. if self._rot_y_axis_reference: @@ -281,9 +220,9 @@ def postprocess( } preds_dicts = ([preds_dict],) - # Build a new dict instead of mutating the caller's metadata (the same sample_meta + # Build a new dict instead of mutating the caller's metadata (the same metadata # may be reused across backends for the same frame). - batch_input_metas = [{**sample_meta, "box_type_3d": sample_meta.get("box_type_3d", LiDARInstance3DBoxes)}] + batch_input_metas = [{**metadata, "box_type_3d": metadata.get("box_type_3d", LiDARInstance3DBoxes)}] with torch.no_grad(): predictions_list = self.pytorch_model.pts_bbox_head.predict_by_feat( @@ -330,7 +269,3 @@ def run_backbone_head(self, spatial_features: torch.Tensor) -> List[torch.Tensor List of 6 head output tensors. """ raise NotImplementedError - - def __repr__(self) -> str: - """Return string representation with class name, device, and backend.""" - return f"{self.__class__.__name__}(device={self.device}, backend={self.backend_type})" diff --git a/deployment/projects/centerpoint/inference/onnx_inference_pipeline.py b/deployment/projects/centerpoint/inference/onnx_inference_pipeline.py index d7eb7e471..ce12ec7dd 100644 --- a/deployment/projects/centerpoint/inference/onnx_inference_pipeline.py +++ b/deployment/projects/centerpoint/inference/onnx_inference_pipeline.py @@ -142,7 +142,7 @@ def run_backbone_head(self, spatial_features: torch.Tensor) -> List[torch.Tensor expected_output_names = [ out.name for out in self._components_cfg.get_component("pts_backbone_neck_head").io.outputs ] - output_names = self.order_head_outputs(onnx_output_names, expected_output_names) + output_names = self.order_outputs_by_config(onnx_output_names, expected_output_names) # Run inference with ordered output names (ONNX Runtime returns outputs in the same order) outputs = self.backbone_head_session.run(output_names, {input_name: input_array}) diff --git a/deployment/projects/centerpoint/inference/tensorrt_inference_pipeline.py b/deployment/projects/centerpoint/inference/tensorrt_inference_pipeline.py index 28132a9c4..fbad94a32 100644 --- a/deployment/projects/centerpoint/inference/tensorrt_inference_pipeline.py +++ b/deployment/projects/centerpoint/inference/tensorrt_inference_pipeline.py @@ -6,22 +6,18 @@ import logging import time -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Tuple import numpy as np import pycuda.autoinit # noqa: F401 -import pycuda.driver as cuda import tensorrt as trt import torch from typing_extensions import override from deployment.config.enums import Backend from deployment.config.schema import ComponentsConfig -from deployment.inference.gpu_resource_mixin import ( - GPUResourceMixin, - TensorRTResourceManager, - release_tensorrt_resources, -) +from deployment.inference.gpu_resource_mixin import GPUResourceMixin, release_tensorrt_resources +from deployment.inference.tensorrt_runner import list_trt_io_names, load_trt_engine, run_trt_engine from deployment.primitives.artifacts import resolve_artifact_path from deployment.primitives.device import DeviceSpec from deployment.projects.centerpoint.inference.centerpoint_inference_pipeline import CenterPointInferencePipeline @@ -39,23 +35,20 @@ class CenterPointTensorRTInferencePipeline(GPUResourceMixin, CenterPointInferenc tensorrt_dir: Directory containing TensorRT engine files. """ - # Free the CUDA cache every N evaluated samples - _GPU_CLEANUP_INTERVAL = 10 - def __init__( self, pytorch_model: torch.nn.Module, tensorrt_dir: str, - components_cfg: ComponentsConfig, device: DeviceSpec, + components_cfg: ComponentsConfig, ) -> None: """Initialize TensorRT pipeline. Args: pytorch_model: Reference PyTorch model for preprocessing. tensorrt_dir: Directory containing TensorRT engine files. - components_cfg: Component configuration from deploy_config (use ComponentsConfig.from_dict). device: Target CUDA device ('cuda:N'). + components_cfg: Component configuration from deploy_config (use ComponentsConfig.from_dict). Raises: ValueError: If device is not a CUDA device or components_cfg is None. @@ -66,7 +59,7 @@ def __init__( self._components_cfg = components_cfg self._engines: Dict[str, trt.ICudaEngine] = {} self._contexts: Dict[str, trt.IExecutionContext] = {} - self._logger = trt.Logger(trt.Logger.WARNING) + self._trt_logger = trt.Logger(trt.Logger.WARNING) # Per-stage pure-GPU times (ms), filled by each stage while its CUDA stream is # still alive and read back in run_model. @@ -82,8 +75,8 @@ def _load_tensorrt_engines(self) -> None: FileNotFoundError: If engine files are not found. RuntimeError: If engine loading or context creation fails. """ - trt.init_libnvinfer_plugins(self._logger, "") - runtime = trt.Runtime(self._logger) + trt.init_libnvinfer_plugins(self._trt_logger, "") + runtime = trt.Runtime(self._trt_logger) engine_files = { "pts_voxel_encoder": resolve_artifact_path( @@ -101,121 +94,11 @@ def _load_tensorrt_engines(self) -> None: } for component_name, engine_path in engine_files.items(): - with open(engine_path, "rb") as f: - engine = runtime.deserialize_cuda_engine(f.read()) - if engine is None: - raise RuntimeError(f"Failed to deserialize engine: {engine_path}") - - context = engine.create_execution_context() - if context is None: - raise RuntimeError( - f"Failed to create execution context for {component_name}. " - "This is likely due to GPU out-of-memory." - ) - + engine, context = load_trt_engine(runtime, engine_path, component_name=component_name) self._engines[component_name] = engine self._contexts[component_name] = context logger.info("Loaded TensorRT engine: %s", component_name) - def _get_io_names( - self, - engine: trt.ICudaEngine, - single_output: bool = False, - ) -> Tuple[str, Union[str, List[str]]]: - """Get input and output tensor names from engine. - - Args: - engine: TensorRT engine. - single_output: If True, return single output name instead of list. - - Returns: - Tuple of (input_name, output_name(s)). - - Raises: - RuntimeError: If input or output names cannot be found. - """ - input_name = None - output_names = [] - - for i in range(engine.num_io_tensors): - tensor_name = engine.get_tensor_name(i) - if engine.get_tensor_mode(tensor_name) == trt.TensorIOMode.INPUT: - input_name = tensor_name - elif engine.get_tensor_mode(tensor_name) == trt.TensorIOMode.OUTPUT: - output_names.append(tensor_name) - - if input_name is None: - raise RuntimeError("Could not find input tensor name") - if not output_names: - raise RuntimeError("Could not find output tensor names") - - if single_output: - return input_name, output_names[0] - return input_name, output_names - - def _run_engine_inference( - self, - context: trt.IExecutionContext, - input_name: str, - input_array: np.ndarray, - output_names: List[str], - ) -> Tuple[Dict[str, np.ndarray], float]: - """Run one TensorRT context end-to-end and return outputs plus pure-GPU time. - - Allocates device buffers, copies the input host->device, executes the context - while timing it with CUDA events, copies every output device->host, and reads - the elapsed GPU time back while the stream is still alive. Shared by the - single-output voxel encoder and the multi-output backbone/head stages. - - Args: - context: Execution context whose engine exposes ``input_name``/``output_names``. - input_name: Engine input tensor name (its shape is set from ``input_array``). - input_array: Contiguous float32 host input. - output_names: Engine output tensor names, in the desired return order. - - Returns: - Tuple of (outputs-by-name as host ndarrays, pure-GPU time in ms). - """ - context.set_input_shape(input_name, input_array.shape) - - # Output shapes can depend on the input shape, so read them only after set_input_shape. - outputs: Dict[str, np.ndarray] = {} - for name in output_names: - output_array = np.empty(context.get_tensor_shape(name), dtype=np.float32) - if not output_array.flags["C_CONTIGUOUS"]: - output_array = np.ascontiguousarray(output_array) - outputs[name] = output_array - - with TensorRTResourceManager() as manager: - d_input = manager.allocate(input_array.nbytes) - d_outputs = {name: manager.allocate(arr.nbytes) for name, arr in outputs.items()} - stream = manager.stream - - context.set_tensor_address(input_name, int(d_input)) - for name in output_names: - context.set_tensor_address(name, int(d_outputs[name])) - - # Memory transfer: CPU -> GPU - cuda.memcpy_htod_async(d_input, input_array, stream) - - # Record start event and execute inference - start_event = cuda.Event() - end_event = cuda.Event() - start_event.record(stream) - context.execute_async_v3(stream_handle=stream.handle) - end_event.record(stream) - - # Memory transfer: GPU -> CPU - for name in output_names: - cuda.memcpy_dtoh_async(outputs[name], d_outputs[name], stream) - manager.synchronize() - - # Read GPU timing while the stream is still alive (events are complete after - # synchronize); avoids reading across a stream that has been released. - gpu_ms = end_event.time_since(start_event) - - return outputs, gpu_ms - @override def run_voxel_encoder(self, input_features: torch.Tensor) -> torch.Tensor: """Run voxel encoder using TensorRT. @@ -235,9 +118,10 @@ def run_voxel_encoder(self, input_features: torch.Tensor) -> torch.Tensor: raise RuntimeError("pts_voxel_encoder context is None - likely failed to initialize due to GPU OOM") input_array = self.to_numpy(input_features, dtype=np.float32) - input_name, output_name = self._get_io_names(engine, single_output=True) + input_names, output_names = list_trt_io_names(engine) + input_name, output_name = input_names[0], output_names[0] - outputs, gpu_ms = self._run_engine_inference(context, input_name, input_array, [output_name]) + outputs, gpu_ms = run_trt_engine(engine, context, {input_name: input_array}, [output_name]) self._gpu_stage_ms["voxel_encoder_ms"] = gpu_ms voxel_features = torch.from_numpy(outputs[output_name]).to(self.torch_device) @@ -263,15 +147,16 @@ def run_backbone_head(self, spatial_features: torch.Tensor) -> List[torch.Tensor raise RuntimeError("pts_backbone_neck_head context is None - likely failed to initialize due to GPU OOM") input_array = self.to_numpy(spatial_features, dtype=np.float32) - input_name, trt_output_names = self._get_io_names(engine, single_output=False) + input_names, trt_output_names = list_trt_io_names(engine) + input_name = input_names[0] expected_output_names = [ out.name for out in self._components_cfg.get_component("pts_backbone_neck_head").io.outputs ] # Validate and order outputs (CenterPoint postprocess depends on the config order). - output_names = self.order_head_outputs(trt_output_names, expected_output_names) + output_names = self.order_outputs_by_config(trt_output_names, expected_output_names) - outputs, gpu_ms = self._run_engine_inference(context, input_name, input_array, output_names) + outputs, gpu_ms = run_trt_engine(engine, context, {input_name: input_array}, output_names) self._gpu_stage_ms["backbone_head_ms"] = gpu_ms return [torch.from_numpy(outputs[name]).to(self.torch_device) for name in output_names] @@ -309,7 +194,7 @@ def run_model( # Stage 2: Middle Encoder (PyTorch, wall-clock). start = time.perf_counter() - spatial_features = self.process_middle_encoder(voxel_features, preprocessed_input["coors"]) + spatial_features = self.run_middle_encoder(voxel_features, preprocessed_input["coors"]) stage_latencies["middle_encoder_ms"] = (time.perf_counter() - start) * 1000 # Stage 3: Backbone + Head (pure-GPU time recorded inside run_backbone_head). @@ -318,12 +203,6 @@ def run_model( return head_outputs, stage_latencies - @override - def periodic_cleanup(self, sample_idx: int) -> None: - """Free the CUDA cache every ``_GPU_CLEANUP_INTERVAL`` samples during long eval loops.""" - if sample_idx > 0 and sample_idx % self._GPU_CLEANUP_INTERVAL == 0 and torch.cuda.is_available(): - torch.cuda.empty_cache() - def _release_gpu_resources(self) -> None: """Release TensorRT resources (engines and contexts).""" release_tensorrt_resources( diff --git a/deployment/projects/centerpoint/io/data_loader.py b/deployment/projects/centerpoint/io/data_loader.py deleted file mode 100644 index 4b21d7f66..000000000 --- a/deployment/projects/centerpoint/io/data_loader.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -CenterPoint DataLoader for deployment. - -Wraps MMDet3D Dataset to ensure GT is identical to tools/detection3d/test.py. -Pipeline is run once per sample in load_sample(), avoiding redundant computation. -""" - -import copy - -import mmdet3d.datasets.transforms # noqa: F401 - registers transforms -import torch -from mmengine.config import Config -from mmengine.registry import DATASETS, init_default_scope -from typing_extensions import override - -from deployment.io.base_data_loader import BaseDataLoader -from deployment.projects.centerpoint.io.sample_types import ( - CenterPointModelInput, - CenterPointSample, -) - - -class CenterPointDataLoader(BaseDataLoader): - """Deployment dataloader for CenterPoint using MMDet3D Dataset. - - This wraps the same Dataset used by tools/detection3d/test.py, ensuring: - - GT is identical - - Pipeline processing is identical - - Pipeline runs once per sample (no cache needed) - - Design: - load_sample() runs the full pipeline and returns all data (input + GT). - preprocess() extracts model inputs from the loaded sample. - """ - - def __init__( - self, - model_cfg: Config, - ) -> None: - """Initialize CenterPoint data loader. - - Args: - model_cfg: MMEngine model config; must have test_dataloader.dataset - (its ``ann_file`` is the test info used for evaluation). - """ - super().__init__() - - self.model_cfg = model_cfg - self.dataset = self._build_dataset(model_cfg) - - def _build_dataset(self, model_cfg: Config) -> torch.utils.data.Dataset: - """Build MMDet3D Dataset from the model config's test_dataloader. - - Args: - model_cfg: MMEngine model config with test_dataloader.dataset. - - Returns: - Built MMDet3D Dataset instance. - - Raises: - AttributeError: If ``model_cfg.test_dataloader`` is missing. - """ - # Set default scope to mmdet3d so transforms are found in the registry - init_default_scope("mmdet3d") - dataset_cfg = copy.deepcopy(model_cfg.test_dataloader.dataset) - - dataset_cfg["test_mode"] = True - - # Build dataset - dataset = DATASETS.build(dataset_cfg) - return dataset - - @override - def load_sample(self, index: int) -> CenterPointSample: - """Load sample by running the full pipeline once. - - Returns a dict containing all data needed for inference and evaluation: - - points: Points tensor (ready for inference) - - metainfo: Sample metadata - - ground_truth: Raw eval_ann_info from MMDet3D (kept unconverted) - - Args: - index: Sample index in the dataset (0 to num_samples - 1). - - Returns: - `CenterPointSample` with keys ``points``, ``metainfo``, ``ground_truth``. - - Raises: - IndexError: If index is out of range. - KeyError: If dataset sample is missing required keys. - ValueError: If ``data_samples`` is None or points shape is invalid. - AttributeError: If ``data_samples`` lacks required attributes. - """ - if index >= len(self.dataset): - raise IndexError(f"Sample index {index} out of range (0-{len(self.dataset)-1})") - - # Run pipeline once - data = self.dataset[index] - - pipeline_inputs = data["inputs"] - points_tensor = pipeline_inputs["points"].to("cpu") - if points_tensor.ndim != 2: - raise ValueError(f"Expected points tensor with shape [N, features], got {points_tensor.shape}") - - data_samples = data["data_samples"] - if data_samples is None: - raise ValueError("Dataset sample contains None 'data_samples', cannot build evaluation ground truth.") - - metainfo = data_samples.metainfo - eval_ann_info = data_samples.eval_ann_info - # Keep raw eval_ann_info here; evaluator will convert to the metrics format. - ground_truth = dict(eval_ann_info) - - return CenterPointSample( - points=points_tensor, - metainfo=dict(metainfo), - ground_truth=ground_truth, - ) - - @override - def preprocess(self, sample: CenterPointSample) -> CenterPointModelInput: - """Extract points and metainfo from loaded sample. - - This is a lightweight operation - pipeline already ran in load_sample(). - - Args: - sample: Result of :meth:`load_sample` with keys ``points`` and ``metainfo``. - - Returns: - Dict with keys ``points`` and ``metainfo`` for inference. - """ - return CenterPointModelInput( - points=sample["points"], - metainfo=sample["metainfo"], - ) - - @property - @override - def num_samples(self) -> int: - """Return the number of samples in the dataset.""" - return len(self.dataset) diff --git a/deployment/projects/centerpoint/io/model_loader.py b/deployment/projects/centerpoint/io/model_loader.py index 39a2ac99f..0c7e160d3 100644 --- a/deployment/projects/centerpoint/io/model_loader.py +++ b/deployment/projects/centerpoint/io/model_loader.py @@ -8,15 +8,16 @@ import copy import logging -from typing import Tuple import torch from mmengine.config import Config -from mmengine.registry import MODELS, init_default_scope -from mmengine.runner import load_checkpoint +from deployment.io.mmdet3d_model import build_mmdet3d_model from deployment.primitives.device import DeviceSpec -from deployment.projects.centerpoint.export.onnx_models import ( # noqa: F401 - register MODELS + +# Imported for their side effect: registering CenterPoint's ONNX module variants into the +# MMDet3D registries so ``MODELS.build`` can resolve them during export. +from deployment.projects.centerpoint.export.onnx_models import ( # noqa: F401 centerpoint_head_onnx, centerpoint_onnx, pillar_encoder_onnx, @@ -69,58 +70,31 @@ def create_onnx_model_cfg( return export_model_cfg -def build_model_from_cfg( +def build_centerpoint_model( model_cfg: Config, checkpoint_path: str, device: DeviceSpec, -) -> torch.nn.Module: - """Build a model from MMEngine config and load checkpoint weights. - - Args: - model_cfg: MMEngine model configuration. - checkpoint_path: Path to the checkpoint file. - device: Target device specification. - - Returns: - Loaded and initialized PyTorch model in eval mode. - """ - # Importing onnx_models above triggers MODELS registration for ONNX variants. - init_default_scope("mmdet3d") - - model_config = copy.deepcopy(model_cfg.model) - model = MODELS.build(model_config) - torch_device = device.to_torch_device() - model.to(torch_device) - load_checkpoint(model, checkpoint_path, map_location=torch_device) - model.eval() - model.cfg = model_cfg - return model - - -def build_centerpoint_onnx_model( - base_model_cfg: Config, - checkpoint_path: str, - device: DeviceSpec, + *, rot_y_axis_reference: bool = False, -) -> Tuple[torch.nn.Module, Config]: - """Build an ONNX-compatible CenterPoint model. +) -> torch.nn.Module: + """Build a CenterPoint model from config and load checkpoint weights (for export + reference eval). - Convenience wrapper that creates ONNX config and builds the model. + Swaps the model config to CenterPoint's ONNX-friendly module variants, then builds and + loads it via the shared mmdet3d model core (mirrors :func:`build_bevfusion_model`). Args: - base_model_cfg: Base MMEngine model configuration. + model_cfg: MMEngine model configuration. checkpoint_path: Path to the checkpoint file. device: Target device specification. rot_y_axis_reference: Whether to use y-axis rotation reference. Returns: - Tuple of ``(model, export_model_cfg)``; the latter matches ``model.cfg``. + The loaded model; the export config it was built from is available as ``model.cfg``. """ export_model_cfg = create_onnx_model_cfg( - base_model_cfg, + model_cfg, device=device, rot_y_axis_reference=rot_y_axis_reference, ) - model = build_model_from_cfg(export_model_cfg, checkpoint_path, device=device) - return model, export_model_cfg + return build_mmdet3d_model(export_model_cfg, checkpoint_path, device) diff --git a/deployment/projects/centerpoint/io/sample_types.py b/deployment/projects/centerpoint/io/sample_types.py index fcc4d8d78..1d24cf926 100644 --- a/deployment/projects/centerpoint/io/sample_types.py +++ b/deployment/projects/centerpoint/io/sample_types.py @@ -1,44 +1,10 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Dict, TypedDict +from typing import TypedDict import torch -from deployment.io.base_data_loader import SampleData - - -class CenterPointSample(SampleData): - """Structured payload after running the MMDet3D test pipeline for one frame. - - Returned by :meth:`deployment.projects.centerpoint.io.data_loader.CenterPointDataLoader.load_sample`. - At runtime this is a plain ``dict``; use bracket access (e.g. ``sample["points"]``). - - Attributes: - points: Point cloud tensor on CPU, shape ``[N, C]`` after pipeline. - metainfo: Per-sample metadata (e.g. lidar path, sample index) as a string-keyed dict. - ground_truth: Raw ``eval_ann_info`` from the detector data sample, for evaluation. - """ - - points: torch.Tensor - metainfo: Dict[str, object] - ground_truth: Dict[str, object] - - -class CenterPointModelInput(TypedDict): - """Subset of a loaded sample passed into the CenterPoint network for inference. - - Produced by :meth:`deployment.projects.centerpoint.io.data_loader.CenterPointDataLoader.preprocess`. - Excludes ``ground_truth``, which is only needed for eval/export wiring. - - Attributes: - points: Point cloud tensor for the model forward. - metainfo: Metadata required by preprocessing or postprocessing. - """ - - points: torch.Tensor - metainfo: Dict[str, object] - class VoxelDict(TypedDict): """Voxelization output from CenterPoint feature extraction (ONNX/export path). diff --git a/deployment/projects/centerpoint/runner.py b/deployment/projects/centerpoint/runner.py index b64346d5b..0049c98c1 100644 --- a/deployment/projects/centerpoint/runner.py +++ b/deployment/projects/centerpoint/runner.py @@ -9,19 +9,18 @@ import torch from mmengine.config import Config +from typing_extensions import override -from deployment.config.base import BaseDeploymentConfig -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.export.contexts import ExportContext +from deployment.evaluation.detection_3d_evaluator import Detection3DEvaluator +from deployment.execution.backend_executor import BackendExecutor from deployment.export.pipelines.onnx_export_pipeline import OnnxExportPipeline from deployment.export.pipelines.tensorrt_export_pipeline import TensorRTExportPipeline from deployment.io.base_data_loader import BaseDataLoader from deployment.primitives.device import DeviceSpec -from deployment.projects.centerpoint.contexts import CenterPointExportContext -from deployment.projects.centerpoint.evaluation.evaluator import CenterPointEvaluator +from deployment.projects.centerpoint.config.centerpoint_deployment_config import CenterPointDeploymentConfig from deployment.projects.centerpoint.export.component_builder import CenterPointComponentBuilder from deployment.projects.centerpoint.export.sample_extractor import CenterPointSampleExtractor -from deployment.projects.centerpoint.io.model_loader import build_centerpoint_onnx_model +from deployment.projects.centerpoint.io.model_loader import build_centerpoint_model from deployment.runtime.runner import BaseDeploymentRunner logger = logging.getLogger(__name__) @@ -41,9 +40,9 @@ class CenterPointDeploymentRunner(BaseDeploymentRunner): def __init__( self, data_loader: BaseDataLoader, - evaluator: CenterPointEvaluator, + evaluator: Detection3DEvaluator, executor: BackendExecutor, - config: BaseDeploymentConfig, + config: CenterPointDeploymentConfig, model_cfg: Config, onnx_pipeline: Optional[OnnxExportPipeline] = None, tensorrt_pipeline: Optional[TensorRTExportPipeline] = None, @@ -68,7 +67,7 @@ def __init__( if onnx_pipeline is None: onnx_pipeline = OnnxExportPipeline( sample_extractor=CenterPointSampleExtractor(), - component_builder=CenterPointComponentBuilder(components_cfg=config.components_cfg), + component_builder=CenterPointComponentBuilder(config=config), ) super().__init__( data_loader=data_loader, @@ -80,36 +79,22 @@ def __init__( tensorrt_pipeline=tensorrt_pipeline, ) - def load_pytorch_model(self, checkpoint_path: str, context: ExportContext) -> torch.nn.Module: + @override + def load_pytorch_model(self, checkpoint_path: str) -> torch.nn.Module: """Load and return the PyTorch model for export. Args: checkpoint_path: Path to the checkpoint file. - context: Export context with additional parameters. Returns: Loaded PyTorch model. """ - rot_y_axis_reference = self._extract_rot_y_axis_reference(context) + rot_y_axis_reference = self.config.rot_y_axis_reference logger.info("Export option rot_y_axis_reference = %s", rot_y_axis_reference) - model, _ = build_centerpoint_onnx_model( - base_model_cfg=self.model_cfg, + return build_centerpoint_model( + model_cfg=self.model_cfg, checkpoint_path=checkpoint_path, device=DeviceSpec.from_value("cpu"), rot_y_axis_reference=rot_y_axis_reference, ) - return model - - def _extract_rot_y_axis_reference(self, context: ExportContext) -> bool: - """Extract rot_y_axis_reference from the export context. - - Args: - context: Export context; must be a ``CenterPointExportContext``. - - Returns: - Boolean value for rot_y_axis_reference. - """ - if not isinstance(context, CenterPointExportContext): - raise TypeError(f"CenterPoint export requires a CenterPointExportContext, got {type(context).__name__}.") - return context.rot_y_axis_reference diff --git a/deployment/projects/registry.py b/deployment/projects/registry.py index ce2a8118a..4cb40e747 100644 --- a/deployment/projects/registry.py +++ b/deployment/projects/registry.py @@ -2,7 +2,6 @@ Project registry for deployment bundles. Each deployment project registers an adapter that knows how to: -- add its CLI args - construct data_loader / evaluator / runner - execute the deployment workflow @@ -13,17 +12,24 @@ import argparse from dataclasses import dataclass -from typing import Callable, Dict, Tuple +from typing import Callable, Dict @dataclass(frozen=True) class ProjectAdapter: - """Minimal adapter interface for a deployment project.""" + """Minimal adapter interface for a deployment project. + + Projects deliberately have no per-project CLI flags: everything that shapes the exported + artifact lives in the deploy config so it is versioned with the artifact and reproducible. + The CLI only carries invocation concerns (``deploy_cfg``, ``model_cfg``, ``--log-level``), + which ``deployment/cli/args.py`` adds to every subparser. + + Required-component validation is the deploy config's job (each project's + ``*DeploymentConfig._validate_components``), so the adapter only maps a name to its ``run``. + """ name: str - add_args: Callable[[argparse.ArgumentParser], None] run: Callable[[argparse.Namespace], int] - required_components: Tuple[str, ...] = () class ProjectRegistry: @@ -53,28 +59,5 @@ def get(self, name: str) -> ProjectAdapter: def list_projects(self) -> list[str]: return sorted(self._adapters.keys()) - def validate_required_components(self, project_name: str, components_cfg) -> None: - """Validate required component keys for a registered project.""" - adapter = self.get(project_name) - if not adapter.required_components: - return - - missing = [] - for component_name in adapter.required_components: - try: - components_cfg.get_component(component_name) - except KeyError: - missing.append(component_name) - - if not missing: - return - - available = sorted(list(components_cfg.component_names())) - missing_str = ", ".join(missing) - available_str = ", ".join(available) - raise KeyError( - f"{adapter.name} requires components [{missing_str}], " f"but available components are [{available_str}]." - ) - project_registry = ProjectRegistry() diff --git a/deployment/runtime/detection3d_entrypoint.py b/deployment/runtime/detection3d_entrypoint.py new file mode 100644 index 000000000..886311c5f --- /dev/null +++ b/deployment/runtime/detection3d_entrypoint.py @@ -0,0 +1,91 @@ +"""Shared entrypoint wiring for 3D-detection deployment projects. + +Every point-cloud 3D detector (CenterPoint, BEVFusion-L, …) wires up its deployment the same +way: parse the two configs, build the typed deploy config, set up file logging, build the +MMDet3D point-cloud data loader, derive the T4MetricV2 metrics config, then hand a shared +``Detection3DEvaluator`` to the project's runner. Only three things vary between projects — the +typed config class, how the backend executor is constructed, and the runner class — so that +variation is injected and the identical wiring lives here once instead of in each ``entrypoint.py``. +""" + +from __future__ import annotations + +import argparse +from typing import Callable + +from mmengine.config import Config + +from deployment.cli.args import add_deployment_file_logging, setup_logging +from deployment.config.base import BaseDeploymentConfig +from deployment.evaluation.detection_3d_evaluator import Detection3DEvaluator +from deployment.execution.backend_executor import BackendExecutor +from deployment.io.point_cloud_data_loader import PointCloudDataLoader +from deployment.metrics.detection_3d_metrics import extract_t4metric_v2_config +from deployment.runtime.runner import BaseDeploymentRunner + +#: Builds the typed deploy config from the raw MMEngine ``deploy_cfg``. +ConfigFactory = Callable[[Config], BaseDeploymentConfig] +#: Builds the project's backend executor from the typed config and the raw ``deploy_cfg`` +#: (the latter carries project-specific extras such as ``tensorrt_config.plugin_libraries``). +ExecutorFactory = Callable[[BaseDeploymentConfig, Config], BackendExecutor] +#: Constructs the project's deployment runner (same keyword contract as ``BaseDeploymentRunner``). +RunnerFactory = Callable[..., BaseDeploymentRunner] + + +def run_detection3d_deployment( + args: argparse.Namespace, + *, + pipeline_name: str, + config_factory: ConfigFactory, + executor_factory: ExecutorFactory, + runner_factory: RunnerFactory, +) -> int: + """Run a 3D-detection deployment workflow with the shared wiring. + + Args: + args: Parsed CLI args carrying ``deploy_cfg``, ``model_cfg`` and ``log_level``. + pipeline_name: Human-readable project name for the log banner (e.g. ``"BEVFusion"``). + config_factory: Builds the typed deploy config from the raw ``deploy_cfg``. + executor_factory: Builds the backend executor from ``(config, deploy_cfg)``. + runner_factory: Builds the deployment runner (``BaseDeploymentRunner`` keyword contract). + + Returns: + Process exit code (0 on success). + """ + logger = setup_logging(args.log_level) + + deploy_cfg = Config.fromfile(args.deploy_cfg) + model_cfg = Config.fromfile(args.model_cfg) + config = config_factory(deploy_cfg) + + log_file = config.resolved_deploy_log_file + if log_file: + add_deployment_file_logging(log_file) + logger.info("Deployment log file: %s", log_file) + + logger.info("=" * 80) + logger.info("%s Deployment Pipeline", pipeline_name) + logger.info("=" * 80) + + # ``runtime_io.info_file`` overrides the dataset's ann_file when set; absent (CenterPoint) it + # is "" and the loader keeps the model config's own ann_file. + info_file = (deploy_cfg.get("runtime_io", {}) or {}).get("info_file", "") + data_loader = PointCloudDataLoader(info_file=info_file, model_cfg=model_cfg) + logger.info("Loaded %s samples", data_loader.num_samples) + + metrics_config = extract_t4metric_v2_config(model_cfg) + + # One executor instance, shared by the evaluator (evaluate/verify) and the runner (which hands + # it the loaded reference model after export). + executor = executor_factory(config, deploy_cfg) + evaluator = Detection3DEvaluator(model_cfg=model_cfg, metrics_config=metrics_config, executor=executor) + runner = runner_factory( + data_loader=data_loader, + evaluator=evaluator, + executor=executor, + config=config, + model_cfg=model_cfg, + ) + + runner.run() + return 0 diff --git a/deployment/runtime/evaluation_orchestrator.py b/deployment/runtime/evaluation_orchestrator.py index 9efae8c73..0cd6287f2 100644 --- a/deployment/runtime/evaluation_orchestrator.py +++ b/deployment/runtime/evaluation_orchestrator.py @@ -12,10 +12,10 @@ from deployment.config.base import BaseDeploymentConfig from deployment.config.enums import Backend from deployment.evaluation.base_evaluator import BaseEvaluator -from deployment.evaluation.evaluator_types import ModelSpec from deployment.inference.gpu_resource_mixin import clear_cuda_memory from deployment.io.base_data_loader import BaseDataLoader from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import ModelSpec from deployment.runtime.artifact_manager import ArtifactManager logger = logging.getLogger(__name__) diff --git a/deployment/runtime/export_orchestrator.py b/deployment/runtime/export_orchestrator.py index e22b92c52..149c1fcf3 100644 --- a/deployment/runtime/export_orchestrator.py +++ b/deployment/runtime/export_orchestrator.py @@ -14,7 +14,6 @@ from deployment.config.base import BaseDeploymentConfig from deployment.config.enums import Backend -from deployment.export.contexts import ExportContext from deployment.export.pipelines.onnx_export_pipeline import OnnxExportPipeline from deployment.export.pipelines.tensorrt_export_pipeline import TensorRTExportPipeline from deployment.io.base_data_loader import BaseDataLoader @@ -82,7 +81,7 @@ def __init__( self._onnx_pipeline = onnx_pipeline self._tensorrt_pipeline = tensorrt_pipeline - def run(self, context: Optional[ExportContext] = None) -> ExportResult: + def run(self) -> ExportResult: """ Execute the complete export workflow. @@ -92,23 +91,16 @@ def run(self, context: Optional[ExportContext] = None) -> ExportResult: 3. Exports to TensorRT if configured 4. Resolves external artifact paths - Args: - context: Typed export context with parameters. If None, a default - ExportContext is created. - Returns: ExportResult containing model and artifact paths """ - if context is None: - context = ExportContext() - result = ExportResult() should_export_onnx = self.config.export_config.should_export_onnx should_export_trt = self.config.export_config.should_export_tensorrt external_onnx_path = self.config.export_config.onnx_path - pytorch_model = self._load_and_register_pytorch_model(self.config.checkpoint_path, context) + pytorch_model = self._load_and_register_pytorch_model(self.config.checkpoint_path) result.pytorch_model = pytorch_model if should_export_onnx: @@ -141,13 +133,12 @@ def run(self, context: Optional[ExportContext] = None) -> ExportResult: self._resolve_external_artifacts(result) return result - def _load_and_register_pytorch_model(self, checkpoint_path: str, context: ExportContext) -> Any: + def _load_and_register_pytorch_model(self, checkpoint_path: str) -> Any: """ Load and register a PyTorch model from checkpoint. Args: checkpoint_path: Path to the PyTorch checkpoint - context: Export context with sample index Returns: Loaded PyTorch model Raises: @@ -155,7 +146,7 @@ def _load_and_register_pytorch_model(self, checkpoint_path: str, context: Export """ logger.info("\nLoading PyTorch model...") try: - pytorch_model = self._model_loader(checkpoint_path, context) + pytorch_model = self._model_loader(checkpoint_path) self.artifact_manager.register_artifact(Backend.PYTORCH, Artifact(path=checkpoint_path)) return pytorch_model except Exception as e: diff --git a/deployment/runtime/runner.py b/deployment/runtime/runner.py index e7207a3f5..ebb6cb439 100644 --- a/deployment/runtime/runner.py +++ b/deployment/runtime/runner.py @@ -16,11 +16,8 @@ from mmengine.config import Config from deployment.config.base import BaseDeploymentConfig -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.evaluation.backend_verifier import BackendVerifier from deployment.evaluation.base_evaluator import BaseEvaluator -from deployment.evaluation.output_comparator import OutputComparator -from deployment.export.contexts import ExportContext +from deployment.execution.backend_executor import BackendExecutor from deployment.export.exporters.model_wrappers import BaseModelWrapper from deployment.export.pipelines.component_builder import DefaultComponentBuilder from deployment.export.pipelines.onnx_export_pipeline import OnnxExportPipeline @@ -31,6 +28,8 @@ from deployment.runtime.evaluation_orchestrator import EvaluationOrchestrator from deployment.runtime.export_orchestrator import ExportOrchestrator from deployment.runtime.verification_orchestrator import VerificationOrchestrator +from deployment.verification.backend_verifier import BackendVerifier +from deployment.verification.output_comparator import OutputComparator logger = logging.getLogger(__name__) @@ -94,16 +93,13 @@ def __init__( self.verification_orchestrator = VerificationOrchestrator(config, verifier, data_loader, self.artifact_manager) self.evaluation_orchestrator = EvaluationOrchestrator(config, evaluator, data_loader, self.artifact_manager) - def load_pytorch_model(self, checkpoint_path: str, context: ExportContext) -> Any: + def load_pytorch_model(self, checkpoint_path: str) -> Any: raise NotImplementedError(f"{self.__class__.__name__}.load_pytorch_model() must be implemented by subclasses.") - def run(self, context: Optional[ExportContext] = None) -> DeploymentResult: - if context is None: - context = ExportContext() - + def run(self) -> DeploymentResult: results = DeploymentResult() - export_result = self.export_orchestrator.run(context) + export_result = self.export_orchestrator.run() results.pytorch_model = export_result.pytorch_model results.onnx_path = export_result.onnx_path results.tensorrt_path = export_result.tensorrt_path diff --git a/deployment/runtime/verification_orchestrator.py b/deployment/runtime/verification_orchestrator.py index 03caf0575..c0471c1c2 100644 --- a/deployment/runtime/verification_orchestrator.py +++ b/deployment/runtime/verification_orchestrator.py @@ -11,10 +11,11 @@ from deployment.config.base import BaseDeploymentConfig from deployment.config.enums import Backend -from deployment.evaluation.backend_verifier import BackendVerifier -from deployment.evaluation.evaluator_types import ModelSpec from deployment.io.base_data_loader import BaseDataLoader +from deployment.primitives.evaluator_types import ModelSpec from deployment.runtime.artifact_manager import ArtifactManager +from deployment.verification.backend_verifier import BackendVerifier +from deployment.verification.reporting import banner logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def __init__( def run(self) -> Dict[str, Any]: """ - Run verification on exported models using policy-based verification. + Run verification on exported models using scenario-based verification. Returns: Verification results dictionary @@ -84,30 +85,30 @@ def run(self) -> Dict[str, Any]: num_verify_samples = verification_cfg.num_verify_samples tolerance = verification_cfg.tolerance - logger.info("=" * 80) + logger.info(banner()) logger.info("Running Verification (mode: %s)", export_mode.value) - logger.info("=" * 80) + logger.info(banner()) all_results: Dict[str, Any] = {} total_passed = 0 total_failed = 0 - for i, policy in enumerate(scenarios): - ref_device = policy.ref_device - test_device = policy.test_device + for i, scenario in enumerate(scenarios): + ref_device = scenario.ref_device + test_device = scenario.test_device logger.info( "\nScenario %s/%s: %s(%s) vs %s(%s)", i + 1, len(scenarios), - policy.ref_backend.value, + scenario.ref_backend.value, ref_device, - policy.test_backend.value, + scenario.test_backend.value, test_device, ) - ref_artifact, ref_valid = self.artifact_manager.resolve_artifact(policy.ref_backend) - test_artifact, test_valid = self.artifact_manager.resolve_artifact(policy.test_backend) + ref_artifact, ref_valid = self.artifact_manager.resolve_artifact(scenario.ref_backend) + test_artifact, test_valid = self.artifact_manager.resolve_artifact(scenario.test_backend) if not ref_valid or not test_valid: ref_path = ref_artifact.path if ref_artifact else None @@ -121,8 +122,8 @@ def run(self) -> Dict[str, Any]: ) continue - reference_spec = ModelSpec(backend=policy.ref_backend, device=ref_device, artifact=ref_artifact) - test_spec = ModelSpec(backend=policy.test_backend, device=test_device, artifact=test_artifact) + reference_spec = ModelSpec(backend=scenario.ref_backend, device=ref_device, artifact=ref_artifact) + test_spec = ModelSpec(backend=scenario.test_backend, device=test_device, artifact=test_artifact) verification_results = self.verifier.run( reference=reference_spec, @@ -132,8 +133,14 @@ def run(self) -> Dict[str, Any]: tolerance=tolerance, ) - policy_key = f"{policy.ref_backend.value}_{ref_device}_vs_{policy.test_backend.value}_{test_device}" - all_results[policy_key] = verification_results + scenario_key = f"{scenario.ref_backend.value}_{ref_device}_vs_{scenario.test_backend.value}_{test_device}" + all_results[scenario_key] = verification_results + + # Surface a pre-inference failure (e.g. device validation) instead of silently + # counting the scenario as 0/0; BackendVerifier sets ``error`` in that case. + if "error" in verification_results: + logger.warning("Scenario %s could not run: %s", i + 1, verification_results["error"]) + continue if "summary" in verification_results: summary = verification_results["summary"] @@ -142,25 +149,25 @@ def run(self) -> Dict[str, Any]: total_passed += passed total_failed += failed if failed == 0: - logger.info("Scenario %s passed (%s comparisons)", i + 1, passed) + logger.info("Scenario %s passed (%s samples)", i + 1, passed) else: logger.warning( - "Scenario %s failed (%s/%s comparisons)", + "Scenario %s failed (%s/%s samples)", i + 1, failed, passed + failed, ) - logger.info("\n" + "=" * 80) + logger.info("\n" + banner()) if total_failed == 0: - logger.info("All verifications passed! (%s total)", total_passed) + logger.info("All verification samples passed! (%s total)", total_passed) else: logger.warning( - "%s/%s verifications failed", + "%s/%s verification samples failed", total_failed, total_passed + total_failed, ) - logger.info("=" * 80) + logger.info(banner()) all_results["summary"] = { "passed": total_passed, diff --git a/deployment/tests/test_centerpoint_configs.py b/deployment/tests/test_centerpoint_configs.py index e47a64def..a6b86ba9c 100644 --- a/deployment/tests/test_centerpoint_configs.py +++ b/deployment/tests/test_centerpoint_configs.py @@ -16,8 +16,6 @@ import pytest from mmengine.config import Config -# Importing the project package registers its ProjectAdapter (required_components). -import deployment.projects.centerpoint # noqa: F401 from deployment.config.schema import ( ComponentsConfig, DeviceConfig, @@ -27,7 +25,6 @@ TensorRTConfig, VerificationConfig, ) -from deployment.projects.registry import project_registry _CONFIG_DIR = Path(__file__).resolve().parents[1] / "projects" / "centerpoint" / "config" _CONFIG_FILES = sorted(_CONFIG_DIR.glob("deploy_config*.py")) @@ -52,15 +49,14 @@ def test_centerpoint_config_parses(config_path: Path) -> None: DeviceConfig.from_dict(cfg.get("devices", {})) # Component keys must be the canonical CenterPoint ids (catches un-renamed outer keys). + # This is the same required-component set that CenterPointDeploymentConfig._validate_components + # enforces at config-construction time. component_names = set(components_cfg.component_names()) assert component_names == { "pts_voxel_encoder", "pts_backbone_neck_head", }, f"{config_path.name}: unexpected component keys {sorted(component_names)}" - # The registry's required-components check is exactly what the entrypoint runs. - project_registry.validate_required_components("centerpoint", components_cfg) - def test_config_dir_is_nonempty() -> None: """Guard against the glob silently matching nothing.""" diff --git a/deployment/tests/test_export_orchestrator.py b/deployment/tests/test_export_orchestrator.py index f284349b8..0f82c7617 100644 --- a/deployment/tests/test_export_orchestrator.py +++ b/deployment/tests/test_export_orchestrator.py @@ -16,7 +16,6 @@ import pytest from deployment.config.enums import Backend -from deployment.export.contexts import ExportContext from deployment.primitives.artifacts import Artifact from deployment.runtime.export_orchestrator import ExportOrchestrator, ExportResult @@ -36,21 +35,21 @@ def test_requested_onnx_producing_nothing_aborts_run(self): export_config = SimpleNamespace(should_export_onnx=True, should_export_tensorrt=True, onnx_path="stale/onnx") orch = _orchestrator(export_config) # Bypass real model loading and force ONNX export to produce nothing. - orch._load_and_register_pytorch_model = lambda ckpt, ctx: object() + orch._load_and_register_pytorch_model = lambda ckpt: object() orch._run_onnx_export = lambda model: None with pytest.raises(RuntimeError, match="stale"): - orch.run(ExportContext()) + orch.run() def test_does_not_reach_tensorrt_when_onnx_fails(self): export_config = SimpleNamespace(should_export_onnx=True, should_export_tensorrt=True, onnx_path="stale/onnx") orch = _orchestrator(export_config) - orch._load_and_register_pytorch_model = lambda ckpt, ctx: object() + orch._load_and_register_pytorch_model = lambda ckpt: object() orch._run_onnx_export = lambda model: None orch._run_tensorrt_export = Mock(side_effect=AssertionError("TensorRT must not run on stale ONNX")) with pytest.raises(RuntimeError): - orch.run(ExportContext()) + orch.run() orch._run_tensorrt_export.assert_not_called() diff --git a/deployment/tests/test_output_comparator.py b/deployment/tests/test_output_comparator.py index 6111ed321..517721bb8 100644 --- a/deployment/tests/test_output_comparator.py +++ b/deployment/tests/test_output_comparator.py @@ -4,7 +4,7 @@ import numpy as np -from deployment.evaluation.output_comparator import OutputComparator +from deployment.verification.output_comparator import OutputComparator class TestOutputComparator: diff --git a/deployment/verification/__init__.py b/deployment/verification/__init__.py new file mode 100644 index 000000000..9fb2f8723 --- /dev/null +++ b/deployment/verification/__init__.py @@ -0,0 +1,6 @@ +"""Cross-backend numerical verification. Import concrete submodules (``deployment.verification.backend_verifier``, …). + +Verification is a peer stage to evaluation (see ``deployment/runtime/verification_orchestrator.py``): +it compares one backend's outputs against another's via ``OutputComparator`` rather than scoring +metrics. It consumes the shared ``deployment.execution`` primitives. +""" diff --git a/deployment/evaluation/backend_verifier.py b/deployment/verification/backend_verifier.py similarity index 87% rename from deployment/evaluation/backend_verifier.py rename to deployment/verification/backend_verifier.py index 9ccaea9c1..2035cc470 100644 --- a/deployment/evaluation/backend_verifier.py +++ b/deployment/verification/backend_verifier.py @@ -17,25 +17,26 @@ import torch from deployment.config.enums import Backend -from deployment.evaluation.backend_executor import BackendExecutor -from deployment.evaluation.evaluator_types import ( +from deployment.execution.backend_executor import BackendExecutor +from deployment.inference.base_inference_pipeline import BaseInferencePipeline +from deployment.io.base_data_loader import BaseDataLoader +from deployment.primitives.device import DeviceSpec +from deployment.primitives.evaluator_types import ( ModelSpec, VerifyResultDict, ) -from deployment.evaluation.output_comparator import ( +from deployment.verification.output_comparator import ( OutputComparator, OutputDiffSummary, TensorDiffDetail, ) -from deployment.inference.base_inference_pipeline import BaseInferencePipeline -from deployment.io.base_data_loader import BaseDataLoader -from deployment.primitives.device import DeviceSpec +from deployment.verification.reporting import banner, format_verdict logger = logging.getLogger(__name__) def _fmt_finite_diff(value: float) -> str: - """Format a diff for logs; ``inf`` is spelled ``inf`` (not ``inf`` via ``%f`` quirks).""" + """Format a diff for logs: literal ``inf`` for infinities, else 6-decimal fixed-point.""" return "inf" if math.isinf(value) else f"{value:.6f}" @@ -177,9 +178,9 @@ def _run_single_sample( """ executor = self._executor - logger.info("\n%s", "=" * 60) + logger.info("\n%s", banner()) logger.info("Verifying sample %s", sample_idx) - logger.info("%s", "=" * 60) + logger.info("%s", banner()) sample = data_loader.load_sample(sample_idx) @@ -234,8 +235,7 @@ def _log_per_output_comparison( ) logger.info(" Overall Max difference: %s", _fmt_finite_diff(summary.max_diff)) logger.info(" Overall Mean difference: %s", _fmt_finite_diff(summary.mean_diff)) - verdict = "PASSED ✓" if summary.passed else "FAILED ✗" - logger.info(" %s verification %s", test_label, verdict) + logger.info(" %s verification %s", test_label, format_verdict(summary.passed)) def _log_header( self, @@ -247,46 +247,40 @@ def _log_header( tolerance: float, ) -> None: """Emit a banner with models, devices, sample count and tolerance.""" - logger.info("\n" + "=" * 60) + logger.info("\n" + banner()) logger.info("Model Verification") - logger.info("=" * 60) + logger.info(banner()) logger.info("Reference: %s on %s - %s", reference.backend.value, ref_device, reference.artifact.path) logger.info("Test: %s on %s - %s", test.backend.value, test_device, test.artifact.path) logger.info("Number of samples: %s", num_samples) logger.info("Tolerance: %s", tolerance) - logger.info("=" * 60) + logger.info(banner()) def _log_sample_result(self, result: SampleVerificationResult) -> None: """Log a single sample's pass/fail verdict plus max/mean diff (and reason on fail).""" - if result.passed: - logger.info( - " sample_%s PASSED ✓ (max_diff=%.6f, mean_diff=%.6f)", - result.sample_idx, - result.max_diff, - result.mean_diff, - ) - else: - logger.warning( - " sample_%s FAILED ✗ (max_diff=%.6f, mean_diff=%.6f) - %s", - result.sample_idx, - result.max_diff, - result.mean_diff, - result.reason or "no diagnostic", - ) + log = logger.info if result.passed else logger.warning + suffix = "" if result.passed else f" - {result.reason or 'no diagnostic'}" + log( + " sample_%s %s (max_diff=%.6f, mean_diff=%.6f)%s", + result.sample_idx, + format_verdict(result.passed), + result.max_diff, + result.mean_diff, + suffix, + ) def _log_summary(self, sample_results: List[SampleVerificationResult]) -> None: """Log per-sample verdicts then an aggregate pass/fail counter.""" - logger.info("\n" + "=" * 60) + logger.info("\n" + banner()) logger.info("Verification Summary") - logger.info("=" * 60) + logger.info(banner()) for r in sample_results: - status = "PASSED" if r.passed else "FAILED" - logger.info(" sample_%s: %s", r.sample_idx, status) + logger.info(" sample_%s: %s", r.sample_idx, format_verdict(r.passed)) total = len(sample_results) passed = sum(1 for r in sample_results if r.passed) failed = total - passed - logger.info("=" * 60) + logger.info(banner()) logger.info("Total: %s/%s passed, %s/%s failed", passed, total, failed, total) - logger.info("=" * 60) + logger.info(banner()) diff --git a/deployment/evaluation/output_comparator.py b/deployment/verification/output_comparator.py similarity index 96% rename from deployment/evaluation/output_comparator.py rename to deployment/verification/output_comparator.py index 760c2c246..50cae4d71 100644 --- a/deployment/evaluation/output_comparator.py +++ b/deployment/verification/output_comparator.py @@ -59,14 +59,12 @@ class TensorDiffDetail: shape: NumPy shape of this tensor. max_diff: Max absolute difference on this tensor. mean_diff: Mean absolute difference on this tensor. - passed: Whether this tensor alone satisfies ``tolerance``. """ path: str shape: Tuple[int, ...] max_diff: float mean_diff: float - passed: bool class OutputComparator: @@ -160,7 +158,6 @@ def _compare_arrays( shape=tuple(int(x) for x in ref_np.shape), max_diff=float("inf"), mean_diff=float("inf"), - passed=False, ) ) return _fail(path, f"shape mismatch {ref_np.shape} vs {test_np.shape}") @@ -180,7 +177,6 @@ def _compare_arrays( shape=tuple(int(x) for x in ref_np.shape), max_diff=max_diff, mean_diff=mean_diff, - passed=passed, ) ) return OutputDiffSummary( @@ -202,7 +198,11 @@ def _merge_summaries(results) -> OutputDiffSummary: for result in results: max_diff = max(max_diff, result.max_diff) - total_diff += result.mean_diff * result.num_elements + # Skip zero-element children (shape/type mismatches carry mean_diff=inf, + # num_elements=0): inf * 0 is nan and would poison the whole mean. The + # mismatch still surfaces via all_passed=False and max_diff=inf. + if result.num_elements: + total_diff += result.mean_diff * result.num_elements total_elements += result.num_elements if not result.passed and all_passed: all_passed = False diff --git a/deployment/verification/reporting.py b/deployment/verification/reporting.py new file mode 100644 index 000000000..de5f67440 --- /dev/null +++ b/deployment/verification/reporting.py @@ -0,0 +1,20 @@ +"""Shared rendering helpers for verification logs. + +One banner width and one pass/fail vocabulary, so the verifier and the verification +orchestrator render results identically instead of each hard-coding their own. +""" + +from __future__ import annotations + +#: Log-banner width used across the verification path (matches the evaluation orchestrator). +BANNER_WIDTH = 80 + + +def banner(char: str = "=") -> str: + """Return a full-width banner line.""" + return char * BANNER_WIDTH + + +def format_verdict(passed: bool) -> str: + """Render a pass/fail verdict token (single source for the ✓/✗ vocabulary).""" + return "PASSED ✓" if passed else "FAILED ✗" diff --git a/projects/BEVFusion/bevfusion/bevfusion_head.py b/projects/BEVFusion/bevfusion/bevfusion_head.py index 2d713b022..f33810737 100644 --- a/projects/BEVFusion/bevfusion/bevfusion_head.py +++ b/projects/BEVFusion/bevfusion/bevfusion_head.py @@ -6,17 +6,17 @@ import torch import torch.nn.functional as F from mmcv.cnn import ConvModule, build_conv_layer -from mmdet3d.models import circle_nms, draw_heatmap_gaussian, gaussian_radius +from mmdet3d.models import draw_heatmap_gaussian, gaussian_radius from mmdet3d.models.dense_heads.centerpoint_head import SeparateHead -from mmdet3d.models.layers import nms_bev from mmdet3d.registry import MODELS -from mmdet3d.structures import xywhr2xyxyr from mmdet.models.task_modules import AssignResult, PseudoSampler, build_assigner, build_bbox_coder, build_sampler from mmdet.models.utils import multi_apply from mmengine.logging import print_log from mmengine.structures import InstanceData from torch import nn +from .utils import apply_cluster_nms + def clip_sigmoid(x, eps=1e-4): y = torch.clamp(x.sigmoid_(), min=eps, max=1 - eps) @@ -444,53 +444,19 @@ def predict_by_feat(self, preds_dicts, metas, img=None, rescale=False, for_roi=F boxes3d = temp[i]["bboxes"] scores = temp[i]["scores"] labels = temp[i]["labels"] - # adopt circle nms for different categories - if self.test_cfg["nms_type"] is not None: - keep_mask = torch.zeros_like(scores) - for nms_cluster in self.nms_clusters: - task_mask = torch.zeros_like(scores) - for cls_idx in nms_cluster["class_indices"]: - task_mask += labels == cls_idx - task_mask = task_mask.bool() - if nms_cluster["nms_threshold"] > 0: - if self.test_cfg["nms_type"] == "circle": - boxes_for_nms = torch.cat( - [ - boxes3d[task_mask][:, :2], - scores[:, None][task_mask], - ], - dim=1, - ) - task_keep_indices = torch.tensor( - circle_nms( - boxes_for_nms.detach().cpu().numpy(), - nms_cluster["nms_threshold"], - post_max_size=nms_cluster["post_max_size"], - ) - ) - else: - boxes_for_nms = xywhr2xyxyr(metas[i]["box_type_3d"](boxes3d[task_mask][:, :7], 7).bev) - top_scores = scores[task_mask] - task_keep_indices = nms_bev( - boxes_for_nms, - top_scores, - thresh=nms_cluster["nms_threshold"], - pre_max_size=self.test_cfg["pre_max_size"], - post_max_size=self.test_cfg["post_max_size"], - ) - else: - task_keep_indices = torch.arange(task_mask.sum()) - if task_keep_indices.shape[0] != 0: - keep_indices = torch.where(task_mask != 0)[0][task_keep_indices] - keep_mask[keep_indices] = 1 - keep_mask = keep_mask.bool() - ret = dict( - bboxes=boxes3d[keep_mask], - scores=scores[keep_mask], - labels=labels[keep_mask], - ) - else: # no nms - ret = dict(bboxes=boxes3d, scores=scores, labels=labels) + # adopt per-cluster nms for different categories (shared with the deployment + # postprocess via apply_cluster_nms so both select the same detections) + boxes3d, scores, labels = apply_cluster_nms( + boxes3d, + scores, + labels, + nms_type=self.test_cfg["nms_type"], + nms_clusters=self.nms_clusters, + box_type_3d=metas[i].get("box_type_3d"), + pre_max_size=self.test_cfg.get("pre_max_size"), + post_max_size=self.test_cfg.get("post_max_size"), + ) + ret = dict(bboxes=boxes3d, scores=scores, labels=labels) temp_instances = InstanceData() temp_instances.bboxes_3d = metas[0]["box_type_3d"](ret["bboxes"], box_dim=ret["bboxes"].shape[-1]) diff --git a/projects/BEVFusion/bevfusion/utils.py b/projects/BEVFusion/bevfusion/utils.py index 39c6a0ded..3ff173ff0 100644 --- a/projects/BEVFusion/bevfusion/utils.py +++ b/projects/BEVFusion/bevfusion/utils.py @@ -7,10 +7,88 @@ except ImportError: linear_sum_assignment = None +from mmdet3d.models import circle_nms +from mmdet3d.models.layers import nms_bev from mmdet3d.registry import TASK_UTILS +from mmdet3d.structures import xywhr2xyxyr from mmengine.structures import InstanceData +def apply_cluster_nms( + boxes3d, + scores, + labels, + nms_type, + nms_clusters, + box_type_3d=None, + pre_max_size=None, + post_max_size=None, +): + """Per-cluster NMS keep-filter shared by the detection head and the deployment postprocess. + + Extracted verbatim from ``BEVFusionHead.predict_by_feat`` so both callers select the same + detections (single source of truth). For each cluster the boxes are kept as-is when the + cluster ``nms_threshold`` is 0, otherwise filtered with circle NMS (``nms_type == 'circle'``, + radius = ``nms_threshold``, capped at the cluster ``post_max_size``) or BEV IoU NMS. + + Args: + boxes3d: Decoded boxes ``[M, >=7]`` (x, y, z, dx, dy, dz, yaw, ...). + scores: Per-box scores ``[M]``. + labels: Per-box class indices ``[M]``. + nms_type: ``'circle'`` for circle NMS, ``None`` to skip NMS, otherwise BEV IoU NMS. + nms_clusters: List of ``{class_indices, nms_threshold, post_max_size}`` dicts. + box_type_3d: Box class used only by the BEV IoU branch (ignored for circle NMS). + pre_max_size: ``test_cfg['pre_max_size']`` for the BEV IoU branch. + post_max_size: ``test_cfg['post_max_size']`` for the BEV IoU branch. + + Returns: + Filtered ``(boxes3d, scores, labels)``. + """ + if nms_type is None: + return boxes3d, scores, labels + + keep_mask = torch.zeros_like(scores) + for nms_cluster in nms_clusters: + task_mask = torch.zeros_like(scores) + for cls_idx in nms_cluster["class_indices"]: + task_mask += labels == cls_idx + task_mask = task_mask.bool() + if nms_cluster["nms_threshold"] > 0: + if nms_type == "circle": + boxes_for_nms = torch.cat( + [ + boxes3d[task_mask][:, :2], + scores[:, None][task_mask], + ], + dim=1, + ) + task_keep_indices = torch.tensor( + circle_nms( + boxes_for_nms.detach().cpu().numpy(), + nms_cluster["nms_threshold"], + post_max_size=nms_cluster["post_max_size"], + ) + ) + else: + boxes_for_nms = xywhr2xyxyr(box_type_3d(boxes3d[task_mask][:, :7], 7).bev) + top_scores = scores[task_mask] + task_keep_indices = nms_bev( + boxes_for_nms, + top_scores, + thresh=nms_cluster["nms_threshold"], + pre_max_size=pre_max_size, + post_max_size=post_max_size, + ) + else: + task_keep_indices = torch.arange(task_mask.sum()) + if task_keep_indices.shape[0] != 0: + keep_indices = torch.where(task_mask != 0)[0][task_keep_indices] + keep_mask[keep_indices] = 1 + keep_mask = keep_mask.bool() + + return boxes3d[keep_mask], scores[keep_mask], labels[keep_mask] + + @TASK_UTILS.register_module() class TransFusionBBoxCoder(BaseBBoxCoder): diff --git a/projects/CenterPoint/README.md b/projects/CenterPoint/README.md index 839fbd498..0b909ee71 100644 --- a/projects/CenterPoint/README.md +++ b/projects/CenterPoint/README.md @@ -141,11 +141,12 @@ where `frame-range` represents the range of frames to visualize. # Deploy for t4dataset (export + verification + evaluation) python -m deployment.cli.main centerpoint \ deployment/projects/centerpoint/config/deploy_config.py \ - projects/CenterPoint/configs/t4dataset/second_secfpn_2xb8_121m_base.py \ - --rot-y-axis-reference + projects/CenterPoint/configs/t4dataset/second_secfpn_2xb8_121m_base.py ``` -where `rot_y_axis_reference` can be removed if we would like to use the original counterclockwise x-axis rotation system. +Set `rot_y_axis_reference = True` in the deploy config to output the y-axis clockwise rotation +system; keep it `False` (the default) for the original counterclockwise x-axis rotation system. +This option lives in the deploy config (not a CLI flag) so it is versioned with the exported artifact. ## Troubleshooting ### Difference from original CenterPoint from mmdetection3d v1