From 695cdd9d293fcd4beb2e58b4e0aec183a49b4ab7 Mon Sep 17 00:00:00 2001 From: jiagaoxiang Date: Thu, 26 Feb 2026 00:19:12 +0000 Subject: [PATCH] Guard rasterization against int32 intersection overflow. Add launch-time checks that reject intersection buffers beyond int32 limits to prevent ROCm VM-fault crashes, and document the detailed root cause plus a 64-bit functional fix path for the team. Made-with: Cursor --- .../debug-int32-intersection-overflow.rst | 119 ++++++++++++++++++ docs/index.rst | 1 + gsplat/cuda/csrc/Rasterization.cpp | 73 +++++++++++ 3 files changed, 193 insertions(+) create mode 100644 docs/how-to/debug-int32-intersection-overflow.rst diff --git a/docs/how-to/debug-int32-intersection-overflow.rst b/docs/how-to/debug-int32-intersection-overflow.rst new file mode 100644 index 0000000..33c28b1 --- /dev/null +++ b/docs/how-to/debug-int32-intersection-overflow.rst @@ -0,0 +1,119 @@ +.. meta:: + :description: Root-cause analysis for int32 intersection overflow VM faults in GSplat rasterization. + :keywords: GSplat, ROCm, VM fault, int32 overflow, rasterize_to_pixels_3dgs_bwd, tile_offsets + +.. _gsplat-int32-intersection-overflow: + +******************************************************************** +Debugging int32 intersection overflow VM faults +******************************************************************** + +Summary +======= + +GSplat rasterization kernels currently consume ``tile_offsets`` and +``flatten_ids`` through 32-bit index paths. When the intersection count +(``n_isects``) exceeds ``INT32_MAX`` (2,147,483,647), offset arithmetic can +wrap and produce negative or otherwise invalid ranges. On ROCm this can +manifest as a GPU VM fault (for example, ``Memory access fault by GPU``), +followed by process abort during device synchronization. + + +Observed failure signature +========================== + +Typical runtime symptom: + +* command runs correctly for smaller scenes +* process aborts for larger scenes at or beyond a threshold +* ROCm reports a memory access fault and aborts the process + +Native backtrace usually shows: + +* ``rocr::core::Runtime::VMFaultHandler(...)`` (abort path) +* HIP synchronization path in ``libamdhip64.so`` +* GSplat launch path leading to + ``rasterize_to_pixels_3dgs_bwd_kernel`` in ``gsplat/csrc.so`` + + +Root cause in detail +==================== + +The backward rasterization kernel reads per-tile intersection ranges from +``tile_offsets`` (``int32_t``) and then indexes into ``flatten_ids``: + +* ``range_start = tile_offsets[tile_id]`` +* ``range_end = tile_offsets[tile_id + 1]`` (or ``n_isects`` for the last tile) +* derived ``idx`` values are used to load ``flatten_ids[idx]`` + +If host-side offsets exceed int32 range and are cast or interpreted as int32, +values wrap by 2^32. This can create invalid ranges and out-of-bounds indices. +Once a kernel thread dereferences an invalid global memory address, ROCm raises +a VM fault. + +In a representative synthetic repro: + +* ``n_isects`` is computed as ``NNZ_PER_CAM * C * ISECTS_PER_GAUSS`` +* threshold crossing occurs near ``C=81`` for constants that place + ``n_isects`` slightly above ``INT32_MAX`` +* offset tensors contain wrapped negative values after int32 conversion +* kernel subsequently faults in backward rasterization + + +Why the current patch is a mitigation +===================================== + +The current patch adds host-side validation before kernel launch and raises a +``TORCH_CHECK`` when ``flatten_ids.numel() > INT32_MAX``. This prevents hard +GPU faults and turns the failure into a deterministic, actionable error. + +This is a safety mitigation, not a full functional fix for very large scenes. + + +Functional fix proposal (recommended) +===================================== + +To support ``n_isects > INT32_MAX`` correctly, migrate intersection indexing to +64-bit end-to-end: + +1. **Public/operator boundary** + + * Accept 64-bit intersection offsets/indices in C++ operator boundaries. + * Keep explicit validation for dtype/range contracts. + +2. **Kernel interfaces** + + * Update kernel signatures and launch parameters to use 64-bit types for: + + * ``n_isects`` + * ``tile_offsets`` + * loop/range/index temporaries derived from offsets + + * Avoid mixed signed/unsigned arithmetic that can silently wrap. + +3. **All rasterization variants** + + * Apply consistently across 3DGS, 2DGS, and world-space rasterization paths + (forward, backward, and index-only kernels) to prevent partial fixes. + +4. **Additional invariants** + + * Validate monotonic non-decreasing offsets. + * Validate final offset against ``flatten_ids.numel()``. + + +Suggested validation plan +========================= + +* Boundary tests around ``INT32_MAX`` (exactly below, equal, above). +* Synthetic stress tests with controlled offset construction. +* ROCm and CUDA parity checks for behavior and numerical correctness. +* Negative tests that verify clean ``TORCH_CHECK`` failures for invalid inputs. + + +Practical guidance until full migration lands +============================================= + +* Keep the fail-fast guard enabled to avoid hard VM faults. +* If large scenes are required immediately, split work into chunks such that + each launch keeps ``n_isects <= INT32_MAX``. diff --git a/docs/index.rst b/docs/index.rst index 878ddb6..48adf41 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,7 @@ The GSplat public repository is located at `https://github.com/ROCm/gsplat ` * :doc:`Benchmarking ` * :doc:`Profiling the library ` + * :doc:`Debug int32 intersection overflow VM faults ` To contribute to the documentation, refer to :doc:`Contribute to GSplat `. diff --git a/gsplat/cuda/csrc/Rasterization.cpp b/gsplat/cuda/csrc/Rasterization.cpp index 45ad828..d1b78e0 100644 --- a/gsplat/cuda/csrc/Rasterization.cpp +++ b/gsplat/cuda/csrc/Rasterization.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,38 @@ namespace gsplat { +namespace { + +inline void validate_intersection_int32_inputs( + const at::Tensor &tile_offsets, + const at::Tensor &flatten_ids, + const char *op_name +) { + TORCH_CHECK( + tile_offsets.scalar_type() == at::kInt, + op_name, + ": tile_offsets must be int32." + ); + TORCH_CHECK( + flatten_ids.scalar_type() == at::kInt, + op_name, + ": flatten_ids must be int32." + ); + + const int64_t n_isects = flatten_ids.numel(); + TORCH_CHECK( + n_isects <= static_cast(std::numeric_limits::max()), + op_name, + ": flatten_ids length (", + n_isects, + ") exceeds int32 limit (", + std::numeric_limits::max(), + "); this overflows tile offsets and can cause GPU memory access faults." + ); +} + +} // namespace + //////////////////////////////////////////////////// // 3DGS @@ -46,6 +79,11 @@ std::tuple rasterize_to_pixels_3dgs_fwd( if (masks.has_value()) { CHECK_INPUT(masks.value()); } + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_pixels_3dgs_fwd" + ); auto opt = means2d.options(); at::DimVector image_dims(tile_offsets.sizes().slice(0, tile_offsets.dim() - 2)); @@ -156,6 +194,11 @@ rasterize_to_pixels_3dgs_bwd( if (masks.has_value()) { CHECK_INPUT(masks.value()); } + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_pixels_3dgs_bwd" + ); uint32_t channels = colors.size(-1); @@ -249,6 +292,11 @@ std::tuple rasterize_to_indices_3dgs( CHECK_INPUT(opacities); CHECK_INPUT(tile_offsets); CHECK_INPUT(flatten_ids); + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_indices_3dgs" + ); auto opt = means2d.options(); uint32_t N = means2d.size(-2); // number of gaussians @@ -355,6 +403,11 @@ rasterize_to_pixels_2dgs_fwd( if (masks.has_value()) { CHECK_INPUT(masks.value()); } + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_pixels_2dgs_fwd" + ); auto opt = means2d.options(); at::DimVector image_dims(tile_offsets.sizes().slice(0, tile_offsets.dim() - 2)); @@ -515,6 +568,11 @@ rasterize_to_pixels_2dgs_bwd( if (masks.has_value()) { CHECK_INPUT(masks.value()); } + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_pixels_2dgs_bwd" + ); uint32_t channels = colors.size(-1); @@ -625,6 +683,11 @@ std::tuple rasterize_to_indices_2dgs( CHECK_INPUT(opacities); CHECK_INPUT(tile_offsets); CHECK_INPUT(flatten_ids); + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_indices_2dgs" + ); auto opt = means2d.options(); uint32_t N = means2d.size(-2); // number of gaussians @@ -735,6 +798,11 @@ std::tuple rasterize_to_pixels_from_world_3d if (masks.has_value()) { CHECK_INPUT(masks.value()); } + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_pixels_from_world_3dgs_fwd" + ); auto opt = means.options(); at::DimVector batch_dims(means.sizes().slice(0, means.dim() - 2)); @@ -872,6 +940,11 @@ rasterize_to_pixels_from_world_3dgs_bwd( if (masks.has_value()) { CHECK_INPUT(masks.value()); } + validate_intersection_int32_inputs( + tile_offsets, + flatten_ids, + "rasterize_to_pixels_from_world_3dgs_bwd" + ); uint32_t channels = colors.size(-1);