From 8ab97c146fa352ca65ec54fb6df7e5946cbffbff Mon Sep 17 00:00:00 2001 From: Hector Date: Wed, 13 Aug 2025 15:27:23 +0200 Subject: [PATCH 1/5] feat(GS): have a manual scale on top of the LOD scale of the splats --- src/materials/shaders/splats.vert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/materials/shaders/splats.vert b/src/materials/shaders/splats.vert index 3da88cf8..44ab1e1d 100644 --- a/src/materials/shaders/splats.vert +++ b/src/materials/shaders/splats.vert @@ -272,7 +272,7 @@ void main() { if(adaptiveSize) { float lodSplatScale = clamp(getLOD( instaceRawPosition, int(vnStart), float(level) ) / maxDepth, 0., 1.); - renderScale = mix(maxSplatScale, 1., lodSplatScale); + renderScale = mix(maxSplatScale * splatScale, 1., lodSplatScale); } vRenderScale = renderScale; From e2c8625be704c518100f2b84fe541633e8b9523b Mon Sep 17 00:00:00 2001 From: Hector Date: Sat, 16 Aug 2025 15:06:25 +0200 Subject: [PATCH 2/5] feat(GS): define positions using a global offset insteaed a local one --- src/loading2/gltf-splats-decoder.ts | 3 ++- src/loading2/gltf-splats-decoder.worker.js | 6 +++--- src/materials/shaders/splats.vert | 6 +++++- src/point-cloud-octree.ts | 3 ++- src/splats-mesh.ts | 7 ++++++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/loading2/gltf-splats-decoder.ts b/src/loading2/gltf-splats-decoder.ts index 86dece84..de9538f5 100644 --- a/src/loading2/gltf-splats-decoder.ts +++ b/src/loading2/gltf-splats-decoder.ts @@ -1,4 +1,4 @@ -import { BufferAttribute, BufferGeometry } from 'three'; +import { BufferAttribute, BufferGeometry, Vector3 } from 'three'; import { GetUrlFn, XhrRequest } from '../loading/types'; import { DecodedGeometry, GeometryDecoder } from './geometry-decoder'; import { OctreeGeometryNode } from './octree-geometry-node'; @@ -234,6 +234,7 @@ export class GltfSplatDecoder implements GeometryDecoder { geometry.userData.maxDepth = this._metadata.hierarchy.depth + 1; geometry.userData.totalSplats = this._metadata.points; + geometry.userData.offset = new Vector3(...offset).sub(min); return { data, buffer, geometry }; } diff --git a/src/loading2/gltf-splats-decoder.worker.js b/src/loading2/gltf-splats-decoder.worker.js index 513bd9b2..aa61b4a5 100644 --- a/src/loading2/gltf-splats-decoder.worker.js +++ b/src/loading2/gltf-splats-decoder.worker.js @@ -456,9 +456,9 @@ onmessage = function (event) { color.z = colors[4 * j + 2]; color.w = colors[4 * j + 3]; - pos.x = positions[4 * j + 0]; - pos.y = positions[4 * j + 1]; - pos.z = positions[4 * j + 2]; + pos.x = rawPositions[4 * j + 0]; + pos.y = rawPositions[4 * j + 1]; + pos.z = rawPositions[4 * j + 2]; let encodedColor = rgbaArrayToInteger([ colX * m0 + color.x * m1, diff --git a/src/materials/shaders/splats.vert b/src/materials/shaders/splats.vert index 44ab1e1d..a9e47877 100644 --- a/src/materials/shaders/splats.vert +++ b/src/materials/shaders/splats.vert @@ -9,6 +9,7 @@ uniform float harmonicsDegree; uniform bool renderIds; uniform bool adaptiveSize; uniform bool renderLoD; +uniform vec3 globalOffset; uniform sampler2D covarianceTexture0; uniform sampler2D covarianceTexture1; @@ -204,6 +205,9 @@ void main() { uvec4 sampledCenterColor = texelFetch(posColorTexture, samplerUV, 0); vec3 instancePosition = uintBitsToFloat(uvec3(sampledCenterColor.gba)); + + instancePosition += globalOffset; + vec3 instaceRawPosition = instancePosition; uint nodeIndex = texelFetch(nodeIndicesTexture, samplerUV, 0).r; @@ -222,7 +226,7 @@ void main() { int vnStart = levelAndVnStart.r; int level = levelAndVnStart.g; - instancePosition += nodeData.rgb; + instaceRawPosition += nodeData.rgb; vec4 viewCenter = modelViewMatrix * vec4(instancePosition, 1.0); vec4 clipCenter = projectionMatrix * viewCenter; diff --git a/src/point-cloud-octree.ts b/src/point-cloud-octree.ts index 3f1b3d5e..3170f677 100644 --- a/src/point-cloud-octree.ts +++ b/src/point-cloud-octree.ts @@ -27,6 +27,7 @@ import { import { computeTransformedBoundingBox } from './utils/bounds'; import { SplatsMesh } from './splats-mesh'; +const DEBUG_MODE = false; export class PointCloudOctree extends PointCloudTree { potree: IPotree; disposed: boolean = false; @@ -173,7 +174,7 @@ export class PointCloudOctree extends PointCloudTree { //Initialise the splats mesh if the nodes contain splats information if (this.renderAsSplats && this.splatsMesh === null) { - this.splatsMesh = new SplatsMesh(false, this.maxAmountOfSplats, this.loadHarmonics); + this.splatsMesh = new SplatsMesh(DEBUG_MODE, this.maxAmountOfSplats, this.loadHarmonics); this.add(this.splatsMesh); } } diff --git a/src/splats-mesh.ts b/src/splats-mesh.ts index ac3d0c91..3a75b0b8 100644 --- a/src/splats-mesh.ts +++ b/src/splats-mesh.ts @@ -249,6 +249,7 @@ export class SplatsMesh extends Object3D { inverseFocalAdjustment: { value: 1 }, splatScale: { value: 1 }, basisViewport: { value: new Vector2(0, 0) }, + globalOffset: { value: new Vector3(0, 0, 0) }, sortedTexture: { value: null }, covarianceTexture0: { value: null }, covarianceTexture1: { value: null }, @@ -267,7 +268,7 @@ export class SplatsMesh extends Object3D { debugMode: { value: false }, renderOnlyHarmonics: { value: false }, renderLoD: { value: false }, - adaptiveSize: { value: true }, + adaptiveSize: { value: false }, harmonicsScale: { value: 4 }, octreeSize: { value: 0 }, fov: { value: 1 }, @@ -380,6 +381,10 @@ export class SplatsMesh extends Object3D { let g = m.geometry as BufferGeometry; if (this.material) { + if (m.name === 'r') { + this.material?.uniforms.globalOffset.value.copy(g.userData.offset); + } + this.material.uniforms.maxDepth.value = g.userData.maxDepth; this.material.uniforms.maxSplatScale.value = g.userData.maxDepth; this.totalSplats = g.userData.totalSplats; From 20339a891967081c2b8367033734c3d51dfe15bf Mon Sep 17 00:00:00 2001 From: Hector Date: Sat, 16 Aug 2025 19:18:38 +0200 Subject: [PATCH 3/5] feat(GS): fix harmonics evaluation --- src/materials/shaders/splats.vert | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/materials/shaders/splats.vert b/src/materials/shaders/splats.vert index a9e47877..434ca584 100644 --- a/src/materials/shaders/splats.vert +++ b/src/materials/shaders/splats.vert @@ -205,10 +205,9 @@ void main() { uvec4 sampledCenterColor = texelFetch(posColorTexture, samplerUV, 0); vec3 instancePosition = uintBitsToFloat(uvec3(sampledCenterColor.gba)); - - instancePosition += globalOffset; - vec3 instaceRawPosition = instancePosition; + vec3 rawPosition = instancePosition; + instancePosition += globalOffset; uint nodeIndex = texelFetch(nodeIndicesTexture, samplerUV, 0).r; @@ -226,7 +225,7 @@ void main() { int vnStart = levelAndVnStart.r; int level = levelAndVnStart.g; - instaceRawPosition += nodeData.rgb; + vec3 instanceNodePosition = instancePosition + nodeData.rgb; vec4 viewCenter = modelViewMatrix * vec4(instancePosition, 1.0); vec4 clipCenter = projectionMatrix * viewCenter; @@ -275,7 +274,7 @@ void main() { float renderScale = 1.; if(adaptiveSize) { - float lodSplatScale = clamp(getLOD( instaceRawPosition, int(vnStart), float(level) ) / maxDepth, 0., 1.); + float lodSplatScale = clamp(getLOD( instanceNodePosition, int(vnStart), float(level) ) / maxDepth, 0., 1.); renderScale = mix(maxSplatScale * splatScale, 1., lodSplatScale); } @@ -301,7 +300,7 @@ void main() { vColor = colorData.rgb; - vec3 worldViewDir = normalize(instancePosition - cameraPosition); + vec3 worldViewDir = normalize(viewCenter.rgb); //Harmonics vec3 harmonics = vec3(0.); @@ -335,9 +334,9 @@ void main() { sh2 = unpack111011s(d2); sh3 = unpack111011s(d3); - float x = worldViewDir.z; + float x = worldViewDir.x; float y = worldViewDir.y; - float z = worldViewDir.x; + float z = worldViewDir.z; float xx = 1.; float yy = 1.; @@ -407,7 +406,6 @@ void main() { SH_C3[4] * x * (4.0 * zz - xx - yy) * sh13 + SH_C3[5] * z * (xx - yy) * sh14 + SH_C3[6] * x * (xx - 3.0 * yy) * sh15; - } } } @@ -422,7 +420,7 @@ void main() { if(renderLoD) { //Test the LOD - int LOD = int(getLOD( instaceRawPosition, int(vnStart), float(level) )); + int LOD = int(getLOD( instanceNodePosition, int(vnStart), float(level) )); switch ( LOD ) { case 0: vColor.rgb = vec3(1., 0., 0.); From 88b3b8e32a793546f5a587f9a09adfcc3c945043 Mon Sep 17 00:00:00 2001 From: Hector Date: Sat, 16 Aug 2025 20:30:41 +0200 Subject: [PATCH 4/5] feat(GS): refactor --- src/materials/shaders/splats.vert | 16 +++++++++------- src/splats-mesh.ts | 25 +++++++++---------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/materials/shaders/splats.vert b/src/materials/shaders/splats.vert index 434ca584..2fc93bed 100644 --- a/src/materials/shaders/splats.vert +++ b/src/materials/shaders/splats.vert @@ -14,6 +14,8 @@ uniform vec3 globalOffset; uniform sampler2D covarianceTexture0; uniform sampler2D covarianceTexture1; uniform sampler2D nodeTexture; +uniform sampler2D nodeTexture2; + uniform highp usampler2D sortedTexture; uniform highp usampler2D posColorTexture; @@ -34,7 +36,6 @@ uniform bool renderOnlyHarmonics; uniform float harmonicsScale; //To read the LOD for each point -uniform highp usampler2D nodeTexture2; uniform sampler2D visibleNodes; uniform float octreeSize; @@ -206,7 +207,7 @@ void main() { uvec4 sampledCenterColor = texelFetch(posColorTexture, samplerUV, 0); vec3 instancePosition = uintBitsToFloat(uvec3(sampledCenterColor.gba)); - vec3 rawPosition = instancePosition; + vec3 nodePosition = instancePosition; instancePosition += globalOffset; uint nodeIndex = texelFetch(nodeIndicesTexture, samplerUV, 0).r; @@ -220,13 +221,14 @@ void main() { samplerUV.x = int(mod(dd, 100.)); vec4 nodeData = texelFetch(nodeTexture, samplerUV, 0); + vec4 nodeData2 = texelFetch(nodeTexture2, samplerUV, 0); + + nodePosition += vec3(nodeData.a, nodeData2.ba); - ivec2 levelAndVnStart = ivec2(texelFetch(nodeTexture2, samplerUV, 0).rg); + ivec2 levelAndVnStart = ivec2(nodeData2.rg); int vnStart = levelAndVnStart.r; int level = levelAndVnStart.g; - vec3 instanceNodePosition = instancePosition + nodeData.rgb; - vec4 viewCenter = modelViewMatrix * vec4(instancePosition, 1.0); vec4 clipCenter = projectionMatrix * viewCenter; vec3 ndcCenter = clipCenter.xyz / clipCenter.w; @@ -274,7 +276,7 @@ void main() { float renderScale = 1.; if(adaptiveSize) { - float lodSplatScale = clamp(getLOD( instanceNodePosition, int(vnStart), float(level) ) / maxDepth, 0., 1.); + float lodSplatScale = clamp(getLOD( nodePosition, int(vnStart), float(level) ) / maxDepth, 0., 1.); renderScale = mix(maxSplatScale * splatScale, 1., lodSplatScale); } @@ -420,7 +422,7 @@ void main() { if(renderLoD) { //Test the LOD - int LOD = int(getLOD( instanceNodePosition, int(vnStart), float(level) )); + int LOD = int(getLOD( nodePosition, int(vnStart), float(level) )); switch ( LOD ) { case 0: vColor.rgb = vec3(1., 0., 0.); diff --git a/src/splats-mesh.ts b/src/splats-mesh.ts index 3a75b0b8..d2c91ae0 100644 --- a/src/splats-mesh.ts +++ b/src/splats-mesh.ts @@ -23,7 +23,6 @@ import { RGFormat, RedIntegerFormat, Texture, - RGIntegerFormat, } from 'three'; import { createSortWorker } from './workers/SortWorker'; @@ -55,7 +54,7 @@ export class SplatsMesh extends Object3D { private bufferCovariance0: Float32Array; private bufferCovariance1: Float32Array; private bufferNodes: Float32Array; - private bufferNodes2: Uint32Array; + private bufferNodes2: Float32Array; private bufferNodesIndices: Uint32Array; private bufferVisibilityNodes: Uint8Array; private bufferHarmonics1: Uint32Array; @@ -122,7 +121,7 @@ export class SplatsMesh extends Object3D { this.bufferCovariance0 = new Float32Array(size * size * 4); this.bufferCovariance1 = new Float32Array(size * size * 2); this.bufferNodes = new Float32Array(100 * 100 * 4); - this.bufferNodes2 = new Uint32Array(100 * 100 * 2); + this.bufferNodes2 = new Float32Array(100 * 100 * 4); this.bufferNodesIndices = new Uint32Array(size * size); this.bufferVisibilityNodes = new Uint8Array(2048 * 4); this.bufferHarmonics1 = new Uint32Array(degree1Size * degree1Size); @@ -131,14 +130,7 @@ export class SplatsMesh extends Object3D { //This should be able to save up to 10000 nodes this.textureNode = new DataTexture(this.bufferNodes, 100, 100, RGBAFormat, FloatType); - this.textureNode2 = new DataTexture( - this.bufferNodes2, - 100, - 100, - RGIntegerFormat, - UnsignedIntType, - ); - this.textureNode2.internalFormat = 'RG32UI'; + this.textureNode2 = new DataTexture(this.bufferNodes2, 100, 100, RGBAFormat, FloatType); this.textureSorted = new DataTexture( this.bufferSorted, @@ -267,7 +259,7 @@ export class SplatsMesh extends Object3D { renderIds: { value: false }, debugMode: { value: false }, renderOnlyHarmonics: { value: false }, - renderLoD: { value: false }, + renderLoD: { value: true }, adaptiveSize: { value: false }, harmonicsScale: { value: 4 }, octreeSize: { value: 0 }, @@ -394,10 +386,11 @@ export class SplatsMesh extends Object3D { const vnStart = pointCloudMaterial.visibleNodeTextureOffsets.get(el.name)!; const level = m.name.length - 1; - let nodeInfo = [m.position.x, m.position.y, m.position.z, 1]; - let nodeInfo2 = [vnStart, level]; + let offset = g.userData.offset; + let nodeInfo = [m.position.x, m.position.y, m.position.z, offset.x]; + let nodeInfo2 = [vnStart, level, offset.y, offset.z]; this.bufferNodes.set(nodeInfo, nodesCount * 4); - this.bufferNodes2.set(nodeInfo2, nodesCount * 2); + this.bufferNodes2.set(nodeInfo2, nodesCount * 4); this.bufferNodesIndices.set( new Uint32Array(g.drawRange.count).fill(nodesCount), @@ -592,7 +585,7 @@ export class SplatsMesh extends Object3D { this.bufferCovariance0 = new Float32Array(0); this.bufferCovariance1 = new Float32Array(0); this.bufferNodes = new Float32Array(0); - this.bufferNodes2 = new Uint32Array(0); + this.bufferNodes2 = new Float32Array(0); this.bufferNodesIndices = new Uint32Array(0); this.bufferVisibilityNodes = new Uint8Array(0); this.bufferHarmonics1 = new Uint32Array(0); From 0a4a0dd1d56592e4c11382c2b9cbd5f4d43a7094 Mon Sep 17 00:00:00 2001 From: Hector Date: Mon, 18 Aug 2025 10:36:15 +0200 Subject: [PATCH 5/5] feat(GS): set default state for the GS shader --- src/splats-mesh.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/splats-mesh.ts b/src/splats-mesh.ts index d2c91ae0..b83d752a 100644 --- a/src/splats-mesh.ts +++ b/src/splats-mesh.ts @@ -259,8 +259,8 @@ export class SplatsMesh extends Object3D { renderIds: { value: false }, debugMode: { value: false }, renderOnlyHarmonics: { value: false }, - renderLoD: { value: true }, - adaptiveSize: { value: false }, + renderLoD: { value: false }, + adaptiveSize: { value: true }, harmonicsScale: { value: 4 }, octreeSize: { value: 0 }, fov: { value: 1 }, @@ -433,6 +433,7 @@ export class SplatsMesh extends Object3D { this.forceSorting = true; this.sortSplats(camera, callback); + return false; } else { return true;