diff --git a/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts b/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts index 2907bec3b09..bbdb0d2acd4 100644 --- a/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts +++ b/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts @@ -148,8 +148,8 @@ export class GaussianSplattingMaterial extends PushMaterial { "focal", "eyePosition", "kernelSize", - "viewDirectionFactor", "alpha", + "depthValues", ]; private _sourceMesh: GaussianSplattingMesh | null = null; /** @@ -328,7 +328,6 @@ export class GaussianSplattingMaterial extends PushMaterial { } effect.setFloat2("focal", focal, focal); - effect.setVector3("viewDirectionFactor", gsMesh.viewDirectionFactor); effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize); effect.setFloat("alpha", gsMaterial.alpha); scene.bindEyePosition(effect, "eyePosition", true); @@ -398,6 +397,84 @@ export class GaussianSplattingMaterial extends PushMaterial { this._afterBind(mesh, this._activeEffect, subMesh); } + protected static _BindEffectUniforms(gsMesh: GaussianSplattingMesh, gsMaterial: GaussianSplattingMaterial, shaderMaterial: ShaderMaterial, scene: Scene): void { + const engine = scene.getEngine(); + const effect = shaderMaterial.getEffect()!; + + gsMesh.getMeshUniformBuffer().bindToEffect(effect, "Mesh"); + shaderMaterial.bindView(effect); + shaderMaterial.bindViewProjection(effect); + + const renderWidth = engine.getRenderWidth(); + const renderHeight = engine.getRenderHeight(); + effect.setFloat2("invViewport", 1 / renderWidth, 1 / renderHeight); + + const projection = scene.getProjectionMatrix(); + const t = projection.m[5]; + const focal = (renderWidth * t) / 2.0; + + effect.setFloat2("focal", focal, focal); + effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize); + effect.setFloat("alpha", gsMaterial.alpha); + + let minZ: number, maxZ: number; + + const camera = scene.activeCamera; + if (!camera) { + return; + } + const cameraIsOrtho = camera.mode === Camera.ORTHOGRAPHIC_CAMERA; + if (cameraIsOrtho) { + minZ = !engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? 0 : 1; + maxZ = engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? 0 : 1; + } else { + minZ = engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? camera.minZ : engine.isNDCHalfZRange ? 0 : camera.minZ; + maxZ = engine.useReverseDepthBuffer && engine.isNDCHalfZRange ? 0 : camera.maxZ; + } + + effect.setFloat2("depthValues", minZ, minZ + maxZ); + + if (gsMesh.covariancesATexture) { + const textureSize = gsMesh.covariancesATexture.getSize(); + effect.setFloat2("dataTextureSize", textureSize.width, textureSize.height); + + effect.setTexture("covariancesATexture", gsMesh.covariancesATexture); + effect.setTexture("covariancesBTexture", gsMesh.covariancesBTexture); + effect.setTexture("centersTexture", gsMesh.centersTexture); + effect.setTexture("colorsTexture", gsMesh.colorsTexture); + } + } + + /** + * Create a depth rendering material for a Gaussian Splatting mesh + * @param scene scene it belongs to + * @param shaderLanguage GLSL or WGSL + * @returns depth rendering shader material + */ + public makeDepthRenderingMaterial(scene: Scene, shaderLanguage: ShaderLanguage): ShaderMaterial { + const shaderMaterial = new ShaderMaterial( + "gaussianSplattingDepthRender", + scene, + { + vertex: "gaussianSplattingDepth", + fragment: "gaussianSplattingDepth", + }, + { + attributes: GaussianSplattingMaterial._Attribs, + uniforms: GaussianSplattingMaterial._Uniforms, + samplers: GaussianSplattingMaterial._Samplers, + uniformBuffers: GaussianSplattingMaterial._UniformBuffers, + shaderLanguage: shaderLanguage, + defines: ["#define DEPTH_RENDER"], + } + ); + shaderMaterial.onBindObservable.add((mesh: AbstractMesh) => { + const gsMaterial = mesh.material as GaussianSplattingMaterial; + const gsMesh = mesh as GaussianSplattingMesh; + GaussianSplattingMaterial._BindEffectUniforms(gsMesh, gsMaterial, shaderMaterial, scene); + }); + return shaderMaterial; + } protected static _MakeGaussianSplattingShadowDepthWrapper(scene: Scene, shaderLanguage: ShaderLanguage): ShadowDepthWrapper { const shaderMaterial = new ShaderMaterial( "gaussianSplattingDepth", @@ -420,34 +497,10 @@ export class GaussianSplattingMaterial extends PushMaterial { }); shaderMaterial.onBindObservable.add((mesh: AbstractMesh) => { - const effect = shaderMaterial.getEffect()!; const gsMaterial = mesh.material as GaussianSplattingMaterial; const gsMesh = mesh as GaussianSplattingMesh; - mesh.getMeshUniformBuffer().bindToEffect(effect, "Mesh"); - shaderMaterial.bindView(effect); - shaderMaterial.bindViewProjection(effect); - - const shadowmapWidth = scene.getEngine().getRenderWidth(); - const shadowmapHeight = scene.getEngine().getRenderHeight(); - effect.setFloat2("invViewport", 1 / shadowmapWidth, 1 / shadowmapHeight); - - const projection = scene.getProjectionMatrix(); - const t = projection.m[5]; - const focal = (shadowmapWidth * t) / 2.0; - - effect.setFloat2("focal", focal, focal); - effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize); - - if (gsMesh.covariancesATexture) { - const textureSize = gsMesh.covariancesATexture.getSize(); - effect.setFloat2("dataTextureSize", textureSize.width, textureSize.height); - - effect.setTexture("covariancesATexture", gsMesh.covariancesATexture); - effect.setTexture("covariancesBTexture", gsMesh.covariancesBTexture); - effect.setTexture("centersTexture", gsMesh.centersTexture); - effect.setTexture("colorsTexture", gsMesh.colorsTexture); - } + GaussianSplattingMaterial._BindEffectUniforms(gsMesh, gsMaterial, shaderMaterial, scene); }); return shadowDepthWrapper; diff --git a/packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts b/packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts index 7d342850bc9..0ead3846049 100644 --- a/packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts +++ b/packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts @@ -133,7 +133,6 @@ export class GaussianSplattingBlock extends NodeMaterialBlock { state._emitUniformFromString("invViewport", NodeMaterialBlockConnectionPointTypes.Vector2); state._emitUniformFromString("kernelSize", NodeMaterialBlockConnectionPointTypes.Float); state._emitUniformFromString("eyePosition", NodeMaterialBlockConnectionPointTypes.Vector3); - state._emitUniformFromString("viewDirectionFactor", NodeMaterialBlockConnectionPointTypes.Vector3); state.attributes.push(VertexBuffer.PositionKind); state.attributes.push("splatIndex0"); state.attributes.push("splatIndex1"); @@ -167,16 +166,14 @@ export class GaussianSplattingBlock extends NodeMaterialBlock { if (state.shaderLanguage === ShaderLanguage.WGSL) { state.compilationString += `let worldRot: mat3x3f = mat3x3f(${world.associatedVariableName}[0].xyz, ${world.associatedVariableName}[1].xyz, ${world.associatedVariableName}[2].xyz);`; state.compilationString += `let normWorldRot: mat3x3f = inverseMat3(worldRot);`; - state.compilationString += `var dir: vec3f = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - uniforms.eyePosition));\n`; - state.compilationString += `dir *= uniforms.viewDirectionFactor;\n`; + state.compilationString += `var eyeToSplatLocalSpace: vec3f = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - uniforms.eyePosition));\n`; } else { state.compilationString += `mat3 worldRot = mat3(${world.associatedVariableName});`; state.compilationString += `mat3 normWorldRot = inverseMat3(worldRot);`; - state.compilationString += `vec3 dir = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - eyePosition));\n`; - state.compilationString += `dir *= viewDirectionFactor;\n`; + state.compilationString += `vec3 eyeToSplatLocalSpace = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - eyePosition));\n`; } - state.compilationString += `${state._declareOutput(sh)} = computeSH(splat, dir);\n`; + state.compilationString += `${state._declareOutput(sh)} = computeSH(splat, eyeToSplatLocalSpace);\n`; state.compilationString += `#else\n`; state.compilationString += `${state._declareOutput(sh)} = vec3${addF}(0.,0.,0.);\n`; state.compilationString += `#endif;\n`; diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index 47bc68ec663..282d44e3b48 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -11,7 +11,6 @@ import { Logger } from "core/Misc/logger"; import { GaussianSplattingMaterial } from "core/Materials/GaussianSplatting/gaussianSplattingMaterial"; import { RawTexture } from "core/Materials/Textures/rawTexture"; import { Constants } from "core/Engines/constants"; -import { Tools } from "core/Misc/tools"; import "core/Meshes/thinInstanceMesh"; import type { ThinEngine } from "core/Engines/thinEngine"; import { ToHalfFloat } from "core/Misc/textureTools"; @@ -20,6 +19,7 @@ import { Scalar } from "core/Maths/math.scalar"; import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler, type Coroutine } from "core/Misc/coroutine"; import { EngineStore } from "core/Engines/engineStore"; import type { Camera } from "core/Cameras/camera"; +import { ImportMeshAsync } from "core/Loading/sceneLoader"; interface IDelayedTextureUpdate { covA: Uint16Array; @@ -28,6 +28,9 @@ interface IDelayedTextureUpdate { centers: Float32Array; sh?: Uint8Array[]; } +interface IUpdateOptions { + flipY?: boolean; +} // @internal const UnpackUnorm = (value: number, bits: number) => { @@ -84,11 +87,6 @@ interface ICompressedPLYChunk { maxColor: Vector3; } -interface IPLYConversionBuffers { - buffer: ArrayBuffer; - sh?: []; -} - /** * To support multiple camera rendering, rendered mesh is separated from the GaussianSplattingMesh itself. * The GS mesh serves as a proxy and a different mesh is rendered for each camera. This hot switch is done @@ -325,15 +323,15 @@ export class GaussianSplattingMesh extends Mesh { // batch size between 2 yield calls during the PLY to splat conversion. private static _PlyConversionBatchSize = 32768; private _shDegree = 0; - private _viewDirectionFactor = new Vector3(1, 1, -1); private static readonly _BatchSize = 16; // 16 splats per instance private _cameraViewInfos = new Map(); /** * View direction factor used to compute the SH view direction in the shader. + * @deprecated Not used anymore for SH rendering */ public get viewDirectionFactor() { - return this._viewDirectionFactor; + return Vector3.OneReadOnly; } /** @@ -420,7 +418,7 @@ export class GaussianSplattingMesh extends Mesh { */ public override set material(value: Material) { this._material = value; - this._material.backFaceCulling = true; + this._material.backFaceCulling = false; this._material.cullBackFaces = false; value.resetDrawCache(); } @@ -1320,15 +1318,14 @@ export class GaussianSplattingMesh extends Mesh { } /** - * Loads a .splat Gaussian or .ply Splatting file asynchronously + * Loads a Gaussian or Splatting file asynchronously * @param url path to the splat file to load + * @param scene optional scene it belongs to * @returns a promise that resolves when the operation is complete * @deprecated Please use SceneLoader.ImportMeshAsync instead */ - public async loadFileAsync(url: string): Promise { - const plyBuffer = await Tools.LoadFileAsync(url, true); - const splatsData: IPLYConversionBuffers = await (GaussianSplattingMesh.ConvertPLYWithSHToSplatAsync(plyBuffer) as any); - await this.updateDataAsync(splatsData.buffer, splatsData.sh); + public async loadFileAsync(url: string, scene?: Scene): Promise { + await ImportMeshAsync(url, (scene || EngineStore.LastCreatedScene)!, { pluginOptions: { splat: { gaussianSplattingMesh: this } } }); } /** @@ -1470,7 +1467,8 @@ export class GaussianSplattingMesh extends Mesh { covB: Uint16Array, colorArray: Uint8Array, minimum: Vector3, - maximum: Vector3 + maximum: Vector3, + options: IUpdateOptions ): void { const matrixRotation = TmpVectors.Matrix[0]; const matrixScale = TmpVectors.Matrix[1]; @@ -1478,7 +1476,7 @@ export class GaussianSplattingMesh extends Mesh { const covBSItemSize = this._useRGBACovariants ? 4 : 2; const x = fBuffer[8 * index + 0]; - const y = -fBuffer[8 * index + 1]; + const y = fBuffer[8 * index + 1] * (options.flipY ? -1 : 1); const z = fBuffer[8 * index + 2]; this._splatPositions![4 * index + 0] = x; @@ -1582,7 +1580,7 @@ export class GaussianSplattingMesh extends Mesh { } } - private *_updateData(data: ArrayBuffer, isAsync: boolean, sh?: Uint8Array[]): Coroutine { + private *_updateData(data: ArrayBuffer, isAsync: boolean, sh?: Uint8Array[], options: IUpdateOptions = { flipY: false }): Coroutine { // if a covariance texture is present, then it's not a creation but an update if (!this._covariancesATexture) { this._readyToDisplay = false; @@ -1630,7 +1628,7 @@ export class GaussianSplattingMesh extends Mesh { const updateLine = partIndex * lineCountUpdate; const splatIndexBase = updateLine * textureSize.x; for (let i = 0; i < textureLengthPerUpdate; i++) { - this._makeSplat(splatIndexBase + i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum); + this._makeSplat(splatIndexBase + i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum, options); } this._updateSubTextures(this._splatPositions, covA, covB, colorArray, updateLine, Math.min(lineCountUpdate, textureSize.y - updateLine)); // Update the binfo @@ -1648,7 +1646,7 @@ export class GaussianSplattingMesh extends Mesh { } else { const paddedVertexCount = (vertexCount + 15) & ~0xf; for (let i = 0; i < vertexCount; i++) { - this._makeSplat(i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum); + this._makeSplat(i, fBuffer, uBuffer, covA, covB, colorArray, minimum, maximum, options); if (isAsync && i % GaussianSplattingMesh._SplatBatchSize === 0) { yield; } @@ -1662,6 +1660,7 @@ export class GaussianSplattingMesh extends Mesh { // Update the binfo this.getBoundingInfo().reConstruct(minimum, maximum, this.getWorldMatrix()); this.setEnabled(true); + this._sortIsDirty = true; } this._postToWorker(true); } @@ -1681,9 +1680,10 @@ export class GaussianSplattingMesh extends Mesh { * Update data from GS (position, orientation, color, scaling) * @param data array that contain all the datas * @param sh optional array of uint8 array for SH data + * @param options optional informations on how to treat data */ - public updateData(data: ArrayBuffer, sh?: Uint8Array[]): void { - runCoroutineSync(this._updateData(data, false, sh)); + public updateData(data: ArrayBuffer, sh?: Uint8Array[], options: IUpdateOptions = { flipY: true }): void { + runCoroutineSync(this._updateData(data, false, sh, options)); } /** diff --git a/packages/dev/core/src/Rendering/depthRenderer.ts b/packages/dev/core/src/Rendering/depthRenderer.ts index d901d8111fc..63aff6afce3 100644 --- a/packages/dev/core/src/Rendering/depthRenderer.ts +++ b/packages/dev/core/src/Rendering/depthRenderer.ts @@ -21,6 +21,7 @@ import { BindBonesParameters, BindMorphTargetParameters, PrepareDefinesAndAttrib import { ShaderLanguage } from "core/Materials/shaderLanguage"; import { EffectFallbacks } from "core/Materials/effectFallbacks"; import type { IEffectCreationOptions } from "core/Materials/effect"; +import type { GaussianSplattingMaterial } from "../Materials/GaussianSplatting/gaussianSplattingMaterial"; /** * This represents a depth renderer in Babylon. @@ -245,7 +246,15 @@ export class DepthRenderer { if (this.isReady(subMesh, hardwareInstancedRendering) && camera) { subMesh._renderId = scene.getRenderId(); - const renderingMaterial = effectiveMesh._internalAbstractMeshDataInfo._materialForRenderPass?.[engine.currentRenderPassId]; + let renderingMaterial = effectiveMesh._internalAbstractMeshDataInfo._materialForRenderPass?.[engine.currentRenderPassId]; + if (renderingMaterial === undefined && effectiveMesh.getClassName() === "GaussianSplattingMesh") { + const gsMaterial = effectiveMesh.material! as GaussianSplattingMaterial; + renderingMaterial = gsMaterial.makeDepthRenderingMaterial(this._scene, this._shaderLanguage); + this.setMaterialForRendering(effectiveMesh, renderingMaterial); + if (!renderingMaterial.isReady()) { + return; + } + } let drawWrapper = subMesh._getDrawWrapper(); if (!drawWrapper && renderingMaterial) { diff --git a/packages/dev/core/src/Shaders/ShadersInclude/gaussianSplatting.fx b/packages/dev/core/src/Shaders/ShadersInclude/gaussianSplatting.fx index 3d8335620c0..a17a3e62122 100644 --- a/packages/dev/core/src/Shaders/ShadersInclude/gaussianSplatting.fx +++ b/packages/dev/core/src/Shaders/ShadersInclude/gaussianSplatting.fx @@ -244,9 +244,7 @@ vec4 gaussianSplatting(vec2 meshPos, vec3 worldPos, vec2 scale, vec3 covA, vec3 ); } - mat3 invy = mat3(1, 0, 0, 0, -1, 0, 0, 0, 1); - - mat3 T = invy * transpose(mat3(modelView)) * J; + mat3 T = transpose(mat3(modelView)) * J; mat3 cov2d = transpose(T) * Vrk * T; #if COMPENSATION diff --git a/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx b/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx index 2cdcbf3d6a7..d7ce3b2e99a 100644 --- a/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx +++ b/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx @@ -16,7 +16,6 @@ uniform vec2 dataTextureSize; uniform vec2 focal; uniform float kernelSize; uniform vec3 eyePosition; -uniform vec3 viewDirectionFactor; uniform float alpha; uniform sampler2D covariancesATexture; @@ -55,9 +54,8 @@ void main () { mat3 worldRot = mat3(world); mat3 normWorldRot = inverseMat3(worldRot); - vec3 dir = normalize(normWorldRot * (worldPos.xyz - eyePosition)); - dir *= viewDirectionFactor; - vColor.xyz = splat.color.xyz + computeSH(splat, dir); + vec3 eyeToSplatLocalSpace = normalize(normWorldRot * (worldPos.xyz - eyePosition)); + vColor.xyz = splat.color.xyz + computeSH(splat, eyeToSplatLocalSpace); #endif vColor.w *= alpha; diff --git a/packages/dev/core/src/Shaders/gaussianSplattingDepth.fragment.fx b/packages/dev/core/src/Shaders/gaussianSplattingDepth.fragment.fx index dafd5f6799a..c6eae531766 100644 --- a/packages/dev/core/src/Shaders/gaussianSplattingDepth.fragment.fx +++ b/packages/dev/core/src/Shaders/gaussianSplattingDepth.fragment.fx @@ -3,13 +3,19 @@ precision highp float; varying vec2 vPosition; varying vec4 vColor; +#ifdef DEPTH_RENDER +varying float vDepthMetric; +#endif + void main(void) { float A = -dot(vPosition, vPosition); - #if defined(SM_SOFTTRANSPARENTSHADOW) && SM_SOFTTRANSPARENTSHADOW == 1 float alpha = exp(A) * vColor.a; if (A < -4.) discard; #else - if (A < -1.) discard; + if (A < -vColor.a) discard; +#endif +#ifdef DEPTH_RENDER + gl_FragColor = vec4(vDepthMetric, 0.0, 0.0, 1.0); #endif } \ No newline at end of file diff --git a/packages/dev/core/src/Shaders/gaussianSplattingDepth.vertex.fx b/packages/dev/core/src/Shaders/gaussianSplattingDepth.vertex.fx index 6edc6477574..3faca121a7e 100644 --- a/packages/dev/core/src/Shaders/gaussianSplattingDepth.vertex.fx +++ b/packages/dev/core/src/Shaders/gaussianSplattingDepth.vertex.fx @@ -4,6 +4,7 @@ uniform vec2 invViewport; uniform vec2 dataTextureSize; uniform vec2 focal; uniform float kernelSize; +uniform float alpha; uniform sampler2D covariancesATexture; uniform sampler2D covariancesBTexture; @@ -14,6 +15,11 @@ varying vec4 vColor; #include +#ifdef DEPTH_RENDER +uniform vec2 depthValues; +varying float vDepthMetric; +#endif + void main(void) { float splatIndex = getSplatIndex(int(position.z + 0.5)); Splat splat = readSplat(splatIndex); @@ -22,5 +28,14 @@ void main(void) { vec4 worldPosGS = world * vec4(splat.center.xyz, 1.0); vPosition = position.xy; vColor = splat.color; + vColor.w *= alpha; gl_Position = gaussianSplatting(position.xy, worldPosGS.xyz, vec2(1.,1.), covA, covB, world, view, projection); +#ifdef DEPTH_RENDER + #ifdef USE_REVERSE_DEPTHBUFFER + vDepthMetric = ((-gl_Position.z + depthValues.x) / (depthValues.y)); + + #else + vDepthMetric = ((gl_Position.z + depthValues.x) / (depthValues.y)); + #endif +#endif } \ No newline at end of file diff --git a/packages/dev/core/src/ShadersWGSL/ShadersInclude/gaussianSplatting.fx b/packages/dev/core/src/ShadersWGSL/ShadersInclude/gaussianSplatting.fx index ae7beb437c5..5cdb898df79 100644 --- a/packages/dev/core/src/ShadersWGSL/ShadersInclude/gaussianSplatting.fx +++ b/packages/dev/core/src/ShadersWGSL/ShadersInclude/gaussianSplatting.fx @@ -297,13 +297,7 @@ fn gaussianSplatting( ); } - let invy = mat3x3( - 1.0, 0.0, 0.0, - 0.0, -1.0, 0.0, - 0.0, 0.0, 1.0 - ); - - let T = invy * transpose(mat3x3( + let T = transpose(mat3x3( modelView[0].xyz, modelView[1].xyz, modelView[2].xyz)) * J; diff --git a/packages/dev/core/src/ShadersWGSL/gaussianSplatting.vertex.fx b/packages/dev/core/src/ShadersWGSL/gaussianSplatting.vertex.fx index 7369dcca100..aab1af85efd 100644 --- a/packages/dev/core/src/ShadersWGSL/gaussianSplatting.vertex.fx +++ b/packages/dev/core/src/ShadersWGSL/gaussianSplatting.vertex.fx @@ -19,7 +19,6 @@ uniform dataTextureSize: vec2f; uniform focal: vec2f; uniform kernelSize: f32; uniform eyePosition: vec3f; -uniform viewDirectionFactor: vec3f; uniform alpha: f32; // textures @@ -59,9 +58,8 @@ fn main(input : VertexInputs) -> FragmentInputs { let worldRot: mat3x3f = mat3x3f(mesh.world[0].xyz, mesh.world[1].xyz, mesh.world[2].xyz); let normWorldRot: mat3x3f = inverseMat3(worldRot); - var dir: vec3f = normalize(normWorldRot * (worldPos.xyz - uniforms.eyePosition.xyz)); - dir *= uniforms.viewDirectionFactor; - vertexOutputs.vColor = vec4f(splat.color.xyz + computeSH(splat, dir), splat.color.w * uniforms.alpha); + var eyeToSplatLocalSpace: vec3f = normalize(normWorldRot * (worldPos.xyz - uniforms.eyePosition.xyz)); + vertexOutputs.vColor = vec4f(splat.color.xyz + computeSH(splat, eyeToSplatLocalSpace), splat.color.w * uniforms.alpha); #else vertexOutputs.vColor = vec4f(splat.color.xyz, splat.color.w * uniforms.alpha); #endif diff --git a/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.fragment.fx b/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.fragment.fx index 7ee5cb6668a..bbd226d716a 100644 --- a/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.fragment.fx +++ b/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.fragment.fx @@ -2,6 +2,10 @@ varying vPosition: vec2f; varying vColor: vec4f; +#ifdef DEPTH_RENDER +varying vDepthMetric: f32; +#endif + // move discard logic to a function to avoid early return issues and parsing/compilation errors with last '} fn checkDiscard(inPosition: vec2f, inColor: vec4f) -> vec4f { var A : f32 = -dot(inPosition, inPosition); @@ -11,11 +15,15 @@ fn checkDiscard(inPosition: vec2f, inColor: vec4f) -> vec4f { discard; } #else - if (A < -1.) { + if (A < -inColor.a) { discard; } #endif +#ifdef DEPTH_RENDER + return vec4f(fragmentInputs.vDepthMetric, 0.0, 0.0, 1.0); +#else return vec4f(inColor.rgb, alpha); +#endif } #define CUSTOM_FRAGMENT_DEFINITIONS diff --git a/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.vertex.fx b/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.vertex.fx index 39ea696741f..c08015a734a 100644 --- a/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.vertex.fx +++ b/packages/dev/core/src/ShadersWGSL/gaussianSplattingDepth.vertex.fx @@ -11,6 +11,7 @@ uniform invViewport: vec2f; uniform dataTextureSize: vec2f; uniform focal: vec2f; uniform kernelSize: f32; +uniform alpha: f32; var covariancesATexture: texture_2d; var covariancesBTexture: texture_2d; @@ -20,6 +21,11 @@ var colorsTexture: texture_2d; varying vPosition: vec2f; varying vColor: vec4f; +#ifdef DEPTH_RENDER +uniform depthValues: vec2f; +varying vDepthMetric: f32; +#endif + #include @vertex @@ -33,5 +39,14 @@ fn main(input : VertexInputs) -> FragmentInputs { let worldPos: vec4f = mesh.world * vec4f(splat.center.xyz, 1.0); vertexOutputs.vPosition = input.position.xy; vertexOutputs.vColor = splat.color; + vertexOutputs.vColor.w *= uniforms.alpha; vertexOutputs.position = gaussianSplatting(input.position.xy, worldPos.xyz, vec2f(1.0, 1.0), covA, covB, mesh.world, scene.view, scene.projection, uniforms.focal, uniforms.invViewport, uniforms.kernelSize); +#ifdef DEPTH_RENDER + #ifdef USE_REVERSE_DEPTHBUFFER + vertexOutputs.vDepthMetric = ((-vertexOutputs.position.z + uniforms.depthValues.x) / (uniforms.depthValues.y)); + + #else + vertexOutputs.vDepthMetric = ((vertexOutputs.position.z + uniforms.depthValues.x) / (uniforms.depthValues.y)); + #endif +#endif } \ No newline at end of file diff --git a/packages/dev/loaders/src/SPLAT/splatFileLoader.ts b/packages/dev/loaders/src/SPLAT/splatFileLoader.ts index 30849f7ac39..dbfb735be6a 100644 --- a/packages/dev/loaders/src/SPLAT/splatFileLoader.ts +++ b/packages/dev/loaders/src/SPLAT/splatFileLoader.ts @@ -204,11 +204,12 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu const makeGSFromParsedSOG = (parsedSOG: IParsedSplat) => { scene._blockEntityCollection = !!this._assetContainer; - const gaussianSplatting = new GaussianSplattingMesh("GaussianSplatting", null, scene, this._loadingOptions.keepInRam); + const gaussianSplatting = this._loadingOptions.gaussianSplattingMesh ?? new GaussianSplattingMesh("GaussianSplatting", null, scene, this._loadingOptions.keepInRam); gaussianSplatting._parentContainer = this._assetContainer; - gaussianSplatting.viewDirectionFactor.set(1, -1, 1); babylonMeshesArray.push(gaussianSplatting); - gaussianSplatting.updateData(parsedSOG.data, parsedSOG.sh); + gaussianSplatting.updateData(parsedSOG.data, parsedSOG.sh, { flipY: false }); + gaussianSplatting.scaling.y *= -1; + gaussianSplatting.computeWorldMatrix(true); scene._blockEntityCollection = false; }; @@ -269,7 +270,8 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu // eslint-disable-next-line @typescript-eslint/no-floating-promises, github/no-then ParseSpz(buffer, scene, this._loadingOptions).then((parsedSPZ) => { scene._blockEntityCollection = !!this._assetContainer; - const gaussianSplatting = new GaussianSplattingMesh("GaussianSplatting", null, scene, this._loadingOptions.keepInRam); + const gaussianSplatting = + this._loadingOptions.gaussianSplattingMesh ?? new GaussianSplattingMesh("GaussianSplatting", null, scene, this._loadingOptions.keepInRam); if (parsedSPZ.trainedWithAntialiasing) { const gsMaterial = gaussianSplatting.material as GaussianSplattingMaterial; gsMaterial.kernelSize = 0.1; @@ -277,7 +279,11 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu } gaussianSplatting._parentContainer = this._assetContainer; babylonMeshesArray.push(gaussianSplatting); - gaussianSplatting.updateData(parsedSPZ.data, parsedSPZ.sh); + gaussianSplatting.updateData(parsedSPZ.data, parsedSPZ.sh, { flipY: false }); + if (!this._loadingOptions.flipY) { + gaussianSplatting.scaling.y *= -1.0; + gaussianSplatting.computeWorldMatrix(true); + } scene._blockEntityCollection = false; this.applyAutoCameraLimits(parsedSPZ, scene); resolve(babylonMeshesArray); @@ -292,13 +298,12 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu switch (parsedPLY.mode) { case Mode.Splat: { - const gaussianSplatting = new GaussianSplattingMesh("GaussianSplatting", null, scene, this._loadingOptions.keepInRam); + const gaussianSplatting = + this._loadingOptions.gaussianSplattingMesh ?? new GaussianSplattingMesh("GaussianSplatting", null, scene, this._loadingOptions.keepInRam); gaussianSplatting._parentContainer = this._assetContainer; babylonMeshesArray.push(gaussianSplatting); - gaussianSplatting.updateData(parsedPLY.data, parsedPLY.sh); - if (parsedPLY.compressed || !parsedPLY.rawSplat) { - gaussianSplatting.viewDirectionFactor.set(-1, -1, 1); - } + gaussianSplatting.updateData(parsedPLY.data, parsedPLY.sh, { flipY: false }); + gaussianSplatting.scaling.y *= -1.0; if (parsedPLY.chirality === "RightHanded") { gaussianSplatting.scaling.y *= -1.0; @@ -315,6 +320,7 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu gaussianSplatting.rotation = new Vector3(-Math.PI / 2, Math.PI, 0); break; } + gaussianSplatting.computeWorldMatrix(true); } break; case Mode.PointCloud: diff --git a/packages/dev/loaders/src/SPLAT/splatLoadingOptions.ts b/packages/dev/loaders/src/SPLAT/splatLoadingOptions.ts index 34359ef1bf0..2203cb1c53f 100644 --- a/packages/dev/loaders/src/SPLAT/splatLoadingOptions.ts +++ b/packages/dev/loaders/src/SPLAT/splatLoadingOptions.ts @@ -1,3 +1,5 @@ +import type { GaussianSplattingMesh } from "core/Meshes/GaussianSplatting/gaussianSplattingMesh"; + /** * Options for loading Gaussian Splatting and PLY files */ @@ -26,4 +28,9 @@ export type SPLATLoadingOptions = { * Disable automatic camera limits from being applied if they exist in the splat file */ disableAutoCameraLimits?: boolean; + + /** + * Mesh that will be used to load data instead of creating a new one + */ + gaussianSplattingMesh?: GaussianSplattingMesh; }; diff --git a/packages/dev/loaders/src/SPLAT/spz.ts b/packages/dev/loaders/src/SPLAT/spz.ts index 4c65a05080f..d45acafffae 100644 --- a/packages/dev/loaders/src/SPLAT/spz.ts +++ b/packages/dev/loaders/src/SPLAT/spz.ts @@ -54,17 +54,11 @@ export function ParseSpz(data: ArrayBuffer, scene: Scene, loadingOptions: SPLATL const rgba = new Uint8ClampedArray(buffer); const rot = new Uint8ClampedArray(buffer); - let coordinateSign = 1; - let quaternionOffset = 0; - if (!loadingOptions.flipY) { - coordinateSign = -1; - quaternionOffset = 255; - } // positions for (let i = 0; i < splatCount; i++) { position[i * 8 + 0] = read24bComponent(ubuf, byteOffset + 0); - position[i * 8 + 1] = coordinateSign * read24bComponent(ubuf, byteOffset + 3); - position[i * 8 + 2] = coordinateSign * read24bComponent(ubuf, byteOffset + 6); + position[i * 8 + 1] = read24bComponent(ubuf, byteOffset + 3); + position[i * 8 + 2] = read24bComponent(ubuf, byteOffset + 6); byteOffset += 9; } @@ -131,9 +125,6 @@ export function ParseSpz(data: ArrayBuffer, scene: Scene, loadingOptions: SPLATL const square = 1 - sumSquares; rotation[iLargest] = Math.sqrt(Math.max(square, 0)); - rotation[1] *= coordinateSign; - rotation[2] *= coordinateSign; - const shuffle = [3, 0, 1, 2]; // shuffle to match the order of the quaternion components in the splat file for (let j = 0; j < 4; j++) { rot[i * 32 + 28 + j] = Math.round(127.5 + rotation[shuffle[j]] * 127.5); @@ -149,8 +140,8 @@ export function ParseSpz(data: ArrayBuffer, scene: Scene, loadingOptions: SPLATL */ for (let i = 0; i < splatCount; i++) { const x = ubuf[byteOffset + 0]; - const y = ubuf[byteOffset + 1] * coordinateSign + quaternionOffset; - const z = ubuf[byteOffset + 2] * coordinateSign + quaternionOffset; + const y = ubuf[byteOffset + 1]; + const z = ubuf[byteOffset + 2]; const nx = x / 127.5 - 1; const ny = y / 127.5 - 1; const nz = z / 127.5 - 1; diff --git a/packages/tools/tests/test/visualization/ReferenceImages/gsplat-compressedply-sh.png b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-compressedply-sh.png new file mode 100644 index 00000000000..a8771c40ea1 Binary files /dev/null and b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-compressedply-sh.png differ diff --git a/packages/tools/tests/test/visualization/ReferenceImages/gsplat-depth.png b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-depth.png new file mode 100644 index 00000000000..34e688d4239 Binary files /dev/null and b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-depth.png differ diff --git a/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-sh.png b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-sh.png new file mode 100644 index 00000000000..d68cde5d9e6 Binary files /dev/null and b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-sh.png differ diff --git a/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-v3.png b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-v3.png index 3753e7c33f9..2e7b5448ab8 100644 Binary files a/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-v3.png and b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-spz-v3.png differ diff --git a/packages/tools/tests/test/visualization/ReferenceImages/gsplat-update-data.png b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-update-data.png new file mode 100644 index 00000000000..66de053260a Binary files /dev/null and b/packages/tools/tests/test/visualization/ReferenceImages/gsplat-update-data.png differ diff --git a/packages/tools/tests/test/visualization/config.json b/packages/tools/tests/test/visualization/config.json index 54286578da3..85a9892c54b 100644 --- a/packages/tools/tests/test/visualization/config.json +++ b/packages/tools/tests/test/visualization/config.json @@ -1,6 +1,30 @@ { "root": "https://cdn.babylonjs.com", "tests": [ + { + "title": "Gaussian Splatting Depth", + "playgroundId": "#V80DRL#12", + "renderCount": 5, + "referenceImage": "gsplat-depth.png" + }, + { + "title": "Gaussian Splatting Compressed ply SH", + "playgroundId": "#U8O4EP#1", + "renderCount": 5, + "referenceImage": "gsplat-compressedply-sh.png" + }, + { + "title": "Gaussian Splatting SPZ SH", + "playgroundId": "#XSNFXP#2", + "renderCount": 5, + "referenceImage": "gsplat-spz-sh.png" + }, + { + "title": "Gaussian Splatting Update Data", + "playgroundId": "#Q0LBM8#2", + "renderCount": 5, + "referenceImage": "gsplat-update-data.png" + }, { "title": "Gaussian Splatting viewports", "playgroundId": "#CG8GO3#1", @@ -20,7 +44,7 @@ }, { "title": "SPZ v3 - Splat", - "playgroundId": "#V7CV8W#3", + "playgroundId": "#V7CV8W#4", "referenceImage": "gsplat-spz-v3.png" }, {