feat(GS): Refactor splat positions encoding

src/loading2/gltf-splats-decoder.ts

@@ -1,4 +1,4 @@
-import { BufferAttribute, BufferGeometry } from 'three';
+import { BufferAttribute, BufferGeometry, Vector3 } from 'three';
 import { GetUrlFn, XhrRequest } from '../loading/types';
 import { DecodedGeometry, GeometryDecoder } from './geometry-decoder';
 import { OctreeGeometryNode } from './octree-geometry-node';
@@ -234,6 +234,7 @@ export class GltfSplatDecoder implements GeometryDecoder {
 
     geometry.userData.maxDepth = this._metadata.hierarchy.depth + 1;
     geometry.userData.totalSplats = this._metadata.points;
+    geometry.userData.offset = new Vector3(...offset).sub(min);
 
     return { data, buffer, geometry };
   }

src/loading2/gltf-splats-decoder.worker.js

@@ -456,9 +456,9 @@ onmessage = function (event) {
       color.z = colors[4 * j + 2];
       color.w = colors[4 * j + 3];
 
-      pos.x = positions[4 * j + 0];
-      pos.y = positions[4 * j + 1];
-      pos.z = positions[4 * j + 2];
+      pos.x = rawPositions[4 * j + 0];
+      pos.y = rawPositions[4 * j + 1];
+      pos.z = rawPositions[4 * j + 2];
 
       let encodedColor = rgbaArrayToInteger([
         colX * m0 + color.x * m1,

src/materials/shaders/splats.vert

@@ -9,10 +9,13 @@ uniform float harmonicsDegree;
 uniform bool renderIds;
 uniform bool adaptiveSize;
 uniform bool renderLoD;
+uniform vec3 globalOffset;
 
 uniform sampler2D covarianceTexture0;
 uniform sampler2D covarianceTexture1;
 uniform sampler2D nodeTexture;
+uniform sampler2D nodeTexture2;
+
 
 uniform highp usampler2D sortedTexture;
 uniform highp usampler2D posColorTexture;
@@ -33,7 +36,6 @@ uniform bool renderOnlyHarmonics;
 uniform float harmonicsScale;
 
 //To read the LOD for each point
-uniform highp usampler2D nodeTexture2;
 uniform sampler2D visibleNodes;
 uniform float octreeSize;
 
@@ -204,7 +206,9 @@ void main() {
 
     uvec4 sampledCenterColor = texelFetch(posColorTexture, samplerUV, 0);
     vec3 instancePosition = uintBitsToFloat(uvec3(sampledCenterColor.gba));
-    vec3 instaceRawPosition = instancePosition;
+
+    vec3 nodePosition = instancePosition;
+    instancePosition += globalOffset;
 
     uint nodeIndex = texelFetch(nodeIndicesTexture, samplerUV, 0).r;
 
@@ -217,13 +221,14 @@ void main() {
     samplerUV.x = int(mod(dd, 100.));
 
     vec4 nodeData = texelFetch(nodeTexture, samplerUV, 0);
+    vec4 nodeData2 = texelFetch(nodeTexture2, samplerUV, 0);
+
+    nodePosition += vec3(nodeData.a, nodeData2.ba);
 
-    ivec2 levelAndVnStart =  ivec2(texelFetch(nodeTexture2, samplerUV, 0).rg);
+    ivec2 levelAndVnStart =  ivec2(nodeData2.rg);
     int vnStart = levelAndVnStart.r;
     int level = levelAndVnStart.g;
 
-    instancePosition += nodeData.rgb;
-
     vec4 viewCenter = modelViewMatrix * vec4(instancePosition, 1.0);
     vec4 clipCenter = projectionMatrix * viewCenter;
     vec3 ndcCenter = clipCenter.xyz / clipCenter.w;
@@ -271,8 +276,8 @@ void main() {
     float renderScale = 1.;
 
     if(adaptiveSize) {
-        float lodSplatScale = clamp(getLOD( instaceRawPosition, int(vnStart), float(level) ) / maxDepth, 0., 1.);
-        renderScale = mix(maxSplatScale, 1., lodSplatScale);
+        float lodSplatScale = clamp(getLOD( nodePosition, int(vnStart), float(level) ) / maxDepth, 0., 1.);
+        renderScale = mix(maxSplatScale * splatScale, 1., lodSplatScale);
     }
 
     vRenderScale = renderScale;
@@ -297,7 +302,7 @@ void main() {
 
     vColor = colorData.rgb;
 
-    vec3 worldViewDir = normalize(instancePosition - cameraPosition);
+    vec3 worldViewDir = normalize(viewCenter.rgb);
 
     //Harmonics
     vec3 harmonics = vec3(0.);
@@ -331,9 +336,9 @@ void main() {
         sh2 = unpack111011s(d2);
         sh3 = unpack111011s(d3);
 
-        float x = worldViewDir.z;
+        float x = worldViewDir.x;
         float y = worldViewDir.y;
-        float z = worldViewDir.x;
+        float z = worldViewDir.z;
 
         float xx = 1.;
         float yy = 1.;
@@ -403,7 +408,6 @@ void main() {
                     SH_C3[4] * x * (4.0 * zz - xx - yy) * sh13 +
                     SH_C3[5] * z * (xx - yy) * sh14 +
                     SH_C3[6] * x * (xx - 3.0 * yy) * sh15;
-
             }
         }
     }
@@ -418,7 +422,7 @@ void main() {
 
     if(renderLoD) {
         //Test the LOD
-        int LOD = int(getLOD( instaceRawPosition, int(vnStart), float(level) ));
+        int LOD = int(getLOD( nodePosition, int(vnStart), float(level) ));
         switch ( LOD ) {
             case 0:
                 vColor.rgb = vec3(1., 0., 0.);

src/point-cloud-octree.ts

@@ -27,6 +27,7 @@ import {
 import { computeTransformedBoundingBox } from './utils/bounds';
 import { SplatsMesh } from './splats-mesh';
 
+const DEBUG_MODE = false;
 export class PointCloudOctree extends PointCloudTree {
   potree: IPotree;
   disposed: boolean = false;
@@ -173,7 +174,7 @@ export class PointCloudOctree extends PointCloudTree {
 
       //Initialise the splats mesh if the nodes contain splats information
       if (this.renderAsSplats && this.splatsMesh === null) {
-        this.splatsMesh = new SplatsMesh(false, this.maxAmountOfSplats, this.loadHarmonics);
+        this.splatsMesh = new SplatsMesh(DEBUG_MODE, this.maxAmountOfSplats, this.loadHarmonics);
         this.add(this.splatsMesh);
       }
     }

src/splats-mesh.ts

@@ -23,7 +23,6 @@ import {
   RGFormat,
   RedIntegerFormat,
   Texture,
-  RGIntegerFormat,
 } from 'three';
 
 import { createSortWorker } from './workers/SortWorker';
@@ -55,7 +54,7 @@ export class SplatsMesh extends Object3D {
   private bufferCovariance0: Float32Array;
   private bufferCovariance1: Float32Array;
   private bufferNodes: Float32Array;
-  private bufferNodes2: Uint32Array;
+  private bufferNodes2: Float32Array;
   private bufferNodesIndices: Uint32Array;
   private bufferVisibilityNodes: Uint8Array;
   private bufferHarmonics1: Uint32Array;
@@ -122,7 +121,7 @@ export class SplatsMesh extends Object3D {
     this.bufferCovariance0 = new Float32Array(size * size * 4);
     this.bufferCovariance1 = new Float32Array(size * size * 2);
     this.bufferNodes = new Float32Array(100 * 100 * 4);
-    this.bufferNodes2 = new Uint32Array(100 * 100 * 2);
+    this.bufferNodes2 = new Float32Array(100 * 100 * 4);
     this.bufferNodesIndices = new Uint32Array(size * size);
     this.bufferVisibilityNodes = new Uint8Array(2048 * 4);
     this.bufferHarmonics1 = new Uint32Array(degree1Size * degree1Size);
@@ -131,14 +130,7 @@ export class SplatsMesh extends Object3D {
 
     //This should be able to save up to 10000 nodes
     this.textureNode = new DataTexture(this.bufferNodes, 100, 100, RGBAFormat, FloatType);
-    this.textureNode2 = new DataTexture(
-      this.bufferNodes2,
-      100,
-      100,
-      RGIntegerFormat,
-      UnsignedIntType,
-    );
-    this.textureNode2.internalFormat = 'RG32UI';
+    this.textureNode2 = new DataTexture(this.bufferNodes2, 100, 100, RGBAFormat, FloatType);
 
     this.textureSorted = new DataTexture(
       this.bufferSorted,
@@ -249,6 +241,7 @@ export class SplatsMesh extends Object3D {
           inverseFocalAdjustment: { value: 1 },
           splatScale: { value: 1 },
           basisViewport: { value: new Vector2(0, 0) },
+          globalOffset: { value: new Vector3(0, 0, 0) },
           sortedTexture: { value: null },
           covarianceTexture0: { value: null },
           covarianceTexture1: { value: null },
@@ -380,6 +373,10 @@ export class SplatsMesh extends Object3D {
         let g = m.geometry as BufferGeometry;
 
         if (this.material) {
+          if (m.name === 'r') {
+            this.material?.uniforms.globalOffset.value.copy(g.userData.offset);
+          }
+
           this.material.uniforms.maxDepth.value = g.userData.maxDepth;
           this.material.uniforms.maxSplatScale.value = g.userData.maxDepth;
           this.totalSplats = g.userData.totalSplats;
@@ -389,10 +386,11 @@ export class SplatsMesh extends Object3D {
         const vnStart = pointCloudMaterial.visibleNodeTextureOffsets.get(el.name)!;
         const level = m.name.length - 1;
 
-        let nodeInfo = [m.position.x, m.position.y, m.position.z, 1];
-        let nodeInfo2 = [vnStart, level];
+        let offset = g.userData.offset;
+        let nodeInfo = [m.position.x, m.position.y, m.position.z, offset.x];
+        let nodeInfo2 = [vnStart, level, offset.y, offset.z];
         this.bufferNodes.set(nodeInfo, nodesCount * 4);
-        this.bufferNodes2.set(nodeInfo2, nodesCount * 2);
+        this.bufferNodes2.set(nodeInfo2, nodesCount * 4);
 
         this.bufferNodesIndices.set(
           new Uint32Array(g.drawRange.count).fill(nodesCount),
@@ -435,6 +433,7 @@ export class SplatsMesh extends Object3D {
       this.forceSorting = true;
 
       this.sortSplats(camera, callback);
+
       return false;
     } else {
       return true;
@@ -587,7 +586,7 @@ export class SplatsMesh extends Object3D {
     this.bufferCovariance0 = new Float32Array(0);
     this.bufferCovariance1 = new Float32Array(0);
     this.bufferNodes = new Float32Array(0);
-    this.bufferNodes2 = new Uint32Array(0);
+    this.bufferNodes2 = new Float32Array(0);
     this.bufferNodesIndices = new Uint32Array(0);
     this.bufferVisibilityNodes = new Uint8Array(0);
     this.bufferHarmonics1 = new Uint32Array(0);