GS Cleanup + depth renderer + more tests

packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts

@@ -148,8 +148,8 @@ export class GaussianSplattingMaterial extends PushMaterial {
         "focal",
         "eyePosition",
         "kernelSize",
-        "viewDirectionFactor",
         "alpha",
+        "depthValues",
     ];
     private _sourceMesh: GaussianSplattingMesh | null = null;
     /**
@@ -328,7 +328,6 @@ export class GaussianSplattingMaterial extends PushMaterial {
         }
 
         effect.setFloat2("focal", focal, focal);
-        effect.setVector3("viewDirectionFactor", gsMesh.viewDirectionFactor);
         effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize);
         effect.setFloat("alpha", gsMaterial.alpha);
         scene.bindEyePosition(effect, "eyePosition", true);
@@ -398,6 +397,84 @@ export class GaussianSplattingMaterial extends PushMaterial {
         this._afterBind(mesh, this._activeEffect, subMesh);
     }
 
+    protected static _BindEffectUniforms(gsMesh: GaussianSplattingMesh, gsMaterial: GaussianSplattingMaterial, shaderMaterial: ShaderMaterial, scene: Scene): void {
+        const engine = scene.getEngine();
+        const effect = shaderMaterial.getEffect()!;
+
+        gsMesh.getMeshUniformBuffer().bindToEffect(effect, "Mesh");
+        shaderMaterial.bindView(effect);
+        shaderMaterial.bindViewProjection(effect);
+
+        const renderWidth = engine.getRenderWidth();
+        const renderHeight = engine.getRenderHeight();
+        effect.setFloat2("invViewport", 1 / renderWidth, 1 / renderHeight);
+
+        const projection = scene.getProjectionMatrix();
+        const t = projection.m[5];
+        const focal = (renderWidth * t) / 2.0;
+
+        effect.setFloat2("focal", focal, focal);
+        effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize);
+        effect.setFloat("alpha", gsMaterial.alpha);
+
+        let minZ: number, maxZ: number;
+
+        const camera = scene.activeCamera;
+        if (!camera) {
+            return;
+        }
+        const cameraIsOrtho = camera.mode === Camera.ORTHOGRAPHIC_CAMERA;
+        if (cameraIsOrtho) {
+            minZ = !engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? 0 : 1;
+            maxZ = engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? 0 : 1;
+        } else {
+            minZ = engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? camera.minZ : engine.isNDCHalfZRange ? 0 : camera.minZ;
+            maxZ = engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? 0 : camera.maxZ;
+        }
+
+        effect.setFloat2("depthValues", minZ, minZ + maxZ);
+
+        if (gsMesh.covariancesATexture) {
+            const textureSize = gsMesh.covariancesATexture.getSize();
+            effect.setFloat2("dataTextureSize", textureSize.width, textureSize.height);
+
+            effect.setTexture("covariancesATexture", gsMesh.covariancesATexture);
+            effect.setTexture("covariancesBTexture", gsMesh.covariancesBTexture);
+            effect.setTexture("centersTexture", gsMesh.centersTexture);
+            effect.setTexture("colorsTexture", gsMesh.colorsTexture);
+        }
+    }
+
+    /**
+     * Create a depth rendering material for a Gaussian Splatting mesh
+     * @param scene scene it belongs to
+     * @param shaderLanguage GLSL or WGSL
+     * @returns depth rendering shader material
+     */
+    public makeDepthRenderingMaterial(scene: Scene, shaderLanguage: ShaderLanguage): ShaderMaterial {
+        const shaderMaterial = new ShaderMaterial(
+            "gaussianSplattingDepthRender",
+            scene,
+            {
+                vertex: "gaussianSplattingDepth",
+                fragment: "gaussianSplattingDepth",
+            },
+            {
+                attributes: GaussianSplattingMaterial._Attribs,
+                uniforms: GaussianSplattingMaterial._Uniforms,
+                samplers: GaussianSplattingMaterial._Samplers,
+                uniformBuffers: GaussianSplattingMaterial._UniformBuffers,
+                shaderLanguage: shaderLanguage,
+                defines: ["#define DEPTH_RENDER"],
+            }
+        );
+        shaderMaterial.onBindObservable.add((mesh: AbstractMesh) => {
+            const gsMaterial = mesh.material as GaussianSplattingMaterial;
+            const gsMesh = mesh as GaussianSplattingMesh;
+            GaussianSplattingMaterial._BindEffectUniforms(gsMesh, gsMaterial, shaderMaterial, scene);
+        });
+        return shaderMaterial;
+    }
     protected static _MakeGaussianSplattingShadowDepthWrapper(scene: Scene, shaderLanguage: ShaderLanguage): ShadowDepthWrapper {
         const shaderMaterial = new ShaderMaterial(
             "gaussianSplattingDepth",
@@ -420,34 +497,10 @@ export class GaussianSplattingMaterial extends PushMaterial {
         });
 
         shaderMaterial.onBindObservable.add((mesh: AbstractMesh) => {
-            const effect = shaderMaterial.getEffect()!;
             const gsMaterial = mesh.material as GaussianSplattingMaterial;
             const gsMesh = mesh as GaussianSplattingMesh;
 
-            mesh.getMeshUniformBuffer().bindToEffect(effect, "Mesh");
-            shaderMaterial.bindView(effect);
-            shaderMaterial.bindViewProjection(effect);
-
-            const shadowmapWidth = scene.getEngine().getRenderWidth();
-            const shadowmapHeight = scene.getEngine().getRenderHeight();
-            effect.setFloat2("invViewport", 1 / shadowmapWidth, 1 / shadowmapHeight);
-
-            const projection = scene.getProjectionMatrix();
-            const t = projection.m[5];
-            const focal = (shadowmapWidth * t) / 2.0;
-
-            effect.setFloat2("focal", focal, focal);
-            effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize);
-
-            if (gsMesh.covariancesATexture) {
-                const textureSize = gsMesh.covariancesATexture.getSize();
-                effect.setFloat2("dataTextureSize", textureSize.width, textureSize.height);
-
-                effect.setTexture("covariancesATexture", gsMesh.covariancesATexture);
-                effect.setTexture("covariancesBTexture", gsMesh.covariancesBTexture);
-                effect.setTexture("centersTexture", gsMesh.centersTexture);
-                effect.setTexture("colorsTexture", gsMesh.colorsTexture);
-            }
+            GaussianSplattingMaterial._BindEffectUniforms(gsMesh, gsMaterial, shaderMaterial, scene);
         });
 
         return shadowDepthWrapper;

packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts

@@ -133,7 +133,6 @@ export class GaussianSplattingBlock extends NodeMaterialBlock {
         state._emitUniformFromString("invViewport", NodeMaterialBlockConnectionPointTypes.Vector2);
         state._emitUniformFromString("kernelSize", NodeMaterialBlockConnectionPointTypes.Float);
         state._emitUniformFromString("eyePosition", NodeMaterialBlockConnectionPointTypes.Vector3);
-        state._emitUniformFromString("viewDirectionFactor", NodeMaterialBlockConnectionPointTypes.Vector3);
         state.attributes.push(VertexBuffer.PositionKind);
         state.attributes.push("splatIndex0");
         state.attributes.push("splatIndex1");
@@ -167,16 +166,14 @@ export class GaussianSplattingBlock extends NodeMaterialBlock {
             if (state.shaderLanguage === ShaderLanguage.WGSL) {
                 state.compilationString += `let worldRot: mat3x3f =  mat3x3f(${world.associatedVariableName}[0].xyz, ${world.associatedVariableName}[1].xyz, ${world.associatedVariableName}[2].xyz);`;
                 state.compilationString += `let normWorldRot: mat3x3f = inverseMat3(worldRot);`;
-                state.compilationString += `var dir: vec3f = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - uniforms.eyePosition));\n`;
-                state.compilationString += `dir *= uniforms.viewDirectionFactor;\n`;
+                state.compilationString += `var eyeToSplatLocalSpace: vec3f = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - uniforms.eyePosition));\n`;
             } else {
                 state.compilationString += `mat3 worldRot = mat3(${world.associatedVariableName});`;
                 state.compilationString += `mat3 normWorldRot = inverseMat3(worldRot);`;
-                state.compilationString += `vec3 dir = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - eyePosition));\n`;
-                state.compilationString += `dir *= viewDirectionFactor;\n`;
+                state.compilationString += `vec3 eyeToSplatLocalSpace = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - eyePosition));\n`;
             }
 
-            state.compilationString += `${state._declareOutput(sh)} = computeSH(splat, dir);\n`;
+            state.compilationString += `${state._declareOutput(sh)} = computeSH(splat, eyeToSplatLocalSpace);\n`;
             state.compilationString += `#else\n`;
             state.compilationString += `${state._declareOutput(sh)} = vec3${addF}(0.,0.,0.);\n`;
             state.compilationString += `#endif;\n`;

packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts

@@ -11,7 +11,6 @@ import { Logger } from "core/Misc/logger";
 import { GaussianSplattingMaterial } from "core/Materials/GaussianSplatting/gaussianSplattingMaterial";
 import { RawTexture } from "core/Materials/Textures/rawTexture";
 import { Constants } from "core/Engines/constants";
-import { Tools } from "core/Misc/tools";
 import "core/Meshes/thinInstanceMesh";
 import type { ThinEngine } from "core/Engines/thinEngine";
 import { ToHalfFloat } from "core/Misc/textureTools";
@@ -20,6 +19,7 @@ import { Scalar } from "core/Maths/math.scalar";
 import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler, type Coroutine } from "core/Misc/coroutine";
 import { EngineStore } from "core/Engines/engineStore";
 import type { Camera } from "core/Cameras/camera";
+import { ImportMeshAsync } from "core/Loading/sceneLoader";
 
 interface IDelayedTextureUpdate {
     covA: Uint16Array;
@@ -28,6 +28,9 @@ interface IDelayedTextureUpdate {
     centers: Float32Array;
     sh?: Uint8Array[];
 }
+interface IUpdateOptions {
+    flipY?: boolean;
+}
 
 // @internal
 const UnpackUnorm = (value: number, bits: number) => {
@@ -84,11 +87,6 @@ interface ICompressedPLYChunk {
     maxColor: Vector3;
 }
 
-interface IPLYConversionBuffers {
-    buffer: ArrayBuffer;
-    sh?: [];
-}
-
 /**
  * To support multiple camera rendering, rendered mesh is separated from the GaussianSplattingMesh itself.
  * The GS mesh serves as a proxy and a different mesh is rendered for each camera. This hot switch is done
@@ -325,15 +323,15 @@ export class GaussianSplattingMesh extends Mesh {
     // batch size between 2 yield calls during the PLY to splat conversion.
     private static _PlyConversionBatchSize = 32768;
     private _shDegree = 0;
-    private _viewDirectionFactor = new Vector3(1, 1, -1);
 
     private static readonly _BatchSize = 16; // 16 splats per instance
     private _cameraViewInfos = new Map<number, ICameraViewInfo>();
     /**
      * View direction factor used to compute the SH view direction in the shader.
+     * @deprecated Not used anymore for SH rendering
      */
     public get viewDirectionFactor() {
-        return this._viewDirectionFactor;
+        return Vector3.OneReadOnly;
     }
 
     /**
@@ -420,7 +418,7 @@ export class GaussianSplattingMesh extends Mesh {
      */
     public override set material(value: Material) {
         this._material = value;
-        this._material.backFaceCulling = true;
+        this._material.backFaceCulling = false;
         this._material.cullBackFaces = false;
         value.resetDrawCache();
     }
@@ -1320,15 +1318,14 @@ export class GaussianSplattingMesh extends Mesh {
     }
 
     /**
-     * Loads a .splat Gaussian or .ply Splatting file asynchronously
+     * Loads a Gaussian or Splatting file asynchronously
      * @param url path to the splat file to load
+     * @param scene optional scene it belongs to
      * @returns a promise that resolves when the operation is complete
      * @deprecated Please use SceneLoader.ImportMeshAsync instead
      */
-    public async loadFileAsync(url: string): Promise<void> {
-        const plyBuffer = await Tools.LoadFileAsync(url, true);
-        const splatsData: IPLYConversionBuffers = await (GaussianSplattingMesh.ConvertPLYWithSHToSplatAsync(plyBuffer) as any);
-        await this.updateDataAsync(splatsData.buffer, splatsData.sh);
+    public async loadFileAsync(url: string, scene?: Scene): Promise<void> {
+        await ImportMeshAsync(url, (scene || EngineStore.LastCreatedScene)!, { pluginOptions: { splat: { gaussianSplattingMesh: this } } });
     }
 
     /**
@@ -1470,15 +1467,16 @@ export class GaussianSplattingMesh extends Mesh {
         covB: Uint16Array,
         colorArray: Uint8Array,
         minimum: Vector3,
-        maximum: Vector3
+        maximum: Vector3,
+        options: IUpdateOptions
     ): void {
         const matrixRotation = TmpVectors.Matrix[0];
         const matrixScale = TmpVectors.Matrix[1];
         const quaternion = TmpVectors.Quaternion[0];
         const covBSItemSize = this._useRGBACovariants ? 4 : 2;
 
         const x = fBuffer[8 * index + 0];
-        const y = -fBuffer[8 * index + 1];
+        const y = fBuffer[8 * index + 1] * (options.flipY ? -1 : 1);
         const z = fBuffer[8 * index + 2];
 
         this._splatPositions![4 * index + 0] = x;
@@ -1582,7 +1580,7 @@ export class GaussianSplattingMesh extends Mesh {
         }
     }
 
-    private *_updateData(data: ArrayBuffer, isAsync: boolean, sh?: Uint8Array[]): Coroutine<void> {
+    private *_updateData(data: ArrayBuffer, isAsync: boolean, sh?: Uint8Array[], options: IUpdateOptions = { flipY: false }): Coroutine<void> {
         // if a covariance texture is present, then it's not a creation but an update
         if (!this._covariancesATexture) {
             this._readyToDisplay = false;
@@ -1630,7 +1628,7 @@ export class GaussianSplattingMesh extends Mesh {
                 const updateLine = partIndex * lineCountUpdate;
                 const splatIndexBase = updateLine * textureSize.x;
                 for (let i = 0; i < textureLengthPerUpdate; i++) {
-                    this._makeSplat(splatIndexBase + i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum);
+                    this._makeSplat(splatIndexBase + i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum, options);
                 }
                 this._updateSubTextures(this._splatPositions, covA, covB, colorArray, updateLine, Math.min(lineCountUpdate, textureSize.y - updateLine));
                 // Update the binfo
@@ -1648,7 +1646,7 @@ export class GaussianSplattingMesh extends Mesh {
         } else {
             const paddedVertexCount = (vertexCount + 15) & ~0xf;
             for (let i = 0; i < vertexCount; i++) {
-                this._makeSplat(i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum);
+                this._makeSplat(i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum, options);
                 if (isAsync && i % GaussianSplattingMesh._SplatBatchSize === 0) {
                     yield;
                 }
@@ -1662,6 +1660,7 @@ export class GaussianSplattingMesh extends Mesh {
             // Update the binfo
             this.getBoundingInfo().reConstruct(minimum, maximum, this.getWorldMatrix());
             this.setEnabled(true);
+            this._sortIsDirty = true;
         }
         this._postToWorker(true);
     }
@@ -1681,9 +1680,10 @@ export class GaussianSplattingMesh extends Mesh {
      * Update data from GS (position, orientation, color, scaling)
      * @param data array that contain all the datas
      * @param sh optional array of uint8 array for SH data
+     * @param options optional informations on how to treat data
      */
-    public updateData(data: ArrayBuffer, sh?: Uint8Array[]): void {
-        runCoroutineSync(this._updateData(data, false, sh));
+    public updateData(data: ArrayBuffer, sh?: Uint8Array[], options: IUpdateOptions = { flipY: true }): void {
+        runCoroutineSync(this._updateData(data, false, sh, options));
     }
 
     /**

packages/dev/core/src/Rendering/depthRenderer.ts

@@ -21,6 +21,7 @@ import { BindBonesParameters, BindMorphTargetParameters, PrepareDefinesAndAttrib
 import { ShaderLanguage } from "core/Materials/shaderLanguage";
 import { EffectFallbacks } from "core/Materials/effectFallbacks";
 import type { IEffectCreationOptions } from "core/Materials/effect";
+import type { GaussianSplattingMaterial } from "../Materials/GaussianSplatting/gaussianSplattingMaterial";
 
 /**
  * This represents a depth renderer in Babylon.
@@ -245,7 +246,15 @@ export class DepthRenderer {
             if (this.isReady(subMesh, hardwareInstancedRendering) && camera) {
                 subMesh._renderId = scene.getRenderId();
 
-                const renderingMaterial = effectiveMesh._internalAbstractMeshDataInfo._materialForRenderPass?.[engine.currentRenderPassId];
+                let renderingMaterial = effectiveMesh._internalAbstractMeshDataInfo._materialForRenderPass?.[engine.currentRenderPassId];
+                if (renderingMaterial === undefined && effectiveMesh.getClassName() === "GaussianSplattingMesh") {
+                    const gsMaterial = effectiveMesh.material! as GaussianSplattingMaterial;
+                    renderingMaterial = gsMaterial.makeDepthRenderingMaterial(this._scene, this._shaderLanguage);
+                    this.setMaterialForRendering(effectiveMesh, renderingMaterial);
+                    if (!renderingMaterial.isReady()) {
+                        return;
+                    }
+                }
 
                 let drawWrapper = subMesh._getDrawWrapper();
                 if (!drawWrapper && renderingMaterial) {

packages/dev/core/src/Shaders/ShadersInclude/gaussianSplatting.fx

@@ -244,9 +244,7 @@ vec4 gaussianSplatting(vec2 meshPos, vec3 worldPos, vec2 scale, vec3 covA, vec3
         );
     }
 
-    mat3 invy = mat3(1, 0, 0, 0, -1, 0, 0, 0, 1);
-
-    mat3 T = invy * transpose(mat3(modelView)) * J;
+    mat3 T = transpose(mat3(modelView)) * J;
     mat3 cov2d = transpose(T) * Vrk * T;
 
 #if COMPENSATION

packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx

@@ -16,7 +16,6 @@ uniform vec2 dataTextureSize;
 uniform vec2 focal;
 uniform float kernelSize;
 uniform vec3 eyePosition;
-uniform vec3 viewDirectionFactor;
 uniform float alpha;
 
 uniform sampler2D covariancesATexture;
@@ -55,9 +54,8 @@ void main () {
     mat3 worldRot = mat3(world);
     mat3 normWorldRot = inverseMat3(worldRot);
 
-    vec3 dir = normalize(normWorldRot * (worldPos.xyz - eyePosition));
-    dir *= viewDirectionFactor;
-    vColor.xyz = splat.color.xyz + computeSH(splat, dir);
+    vec3 eyeToSplatLocalSpace = normalize(normWorldRot * (worldPos.xyz - eyePosition));
+    vColor.xyz = splat.color.xyz + computeSH(splat, eyeToSplatLocalSpace);
 #endif
     vColor.w *= alpha;
 

packages/dev/core/src/Shaders/gaussianSplattingDepth.fragment.fx

@@ -3,13 +3,19 @@ precision highp float;
 varying vec2 vPosition;
 varying vec4 vColor;
 
+#ifdef DEPTH_RENDER
+varying float vDepthMetric;
+#endif
+
 void main(void) {
     float A = -dot(vPosition, vPosition);
-
 #if defined(SM_SOFTTRANSPARENTSHADOW) && SM_SOFTTRANSPARENTSHADOW == 1
     float alpha = exp(A) * vColor.a;
     if (A < -4.) discard;
 #else
-    if (A < -1.) discard;
+    if (A < -vColor.a) discard;
+#endif
+#ifdef DEPTH_RENDER
+    gl_FragColor = vec4(vDepthMetric, 0.0, 0.0, 1.0);
 #endif
 }