diff --git a/README.md b/README.md index 952caa18..dc17746a 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,23 @@ When I started, web-based viewers were already available -- A WebGL-based viewer - Custom `.ksplat` file format still needs work, especially around compression - The default, integer based splat sort does not work well for larger scenes. In that case a value of `false` for the `integerBasedSort` viewer parameter can force a slower, floating-point based sort +## Limitations + +Currently there are limits on the number of splats that can be rendered, and those limits depend mainly on the degree of spherical harmonics desired. Those limits are: + +| Spherical harmonics degree | Max splat count +| --- | --- +| `0` | ~ 16,000,000 +| `1` | ~ 11,000,000 +| `2` | ~ 8,000,000 + +Future work will include optimizing how splat data is packed into data textures, which will help increase these limits. + ## Future work This is still very much a work in progress! There are several things that still need to be done: - - Improve the method by which splat data is stored in textures + - Improve the way splat data is packed into data textures - Continue optimizing CPU-based splat sort - maybe try an incremental sort of some kind? - - Add editing mode, allowing users to modify scene and export changes - - Support very large scenes + - Support very large scenes (streaming sections & LOD) ## Online demo [https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php](https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php) @@ -318,6 +329,7 @@ Advanced `Viewer` parameters | `enableOptionalEffects` | When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. Default is `false` for performance reasons. These properties are separate from transform properties (scale, rotation, position) that are enabled by the `dynamicScene` parameter. | `plyInMemoryCompressionLevel` | Level to compress `.ply` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 2. | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. +| `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/
### Creating KSPLAT files diff --git a/package-lock.json b/package-lock.json index fa50e288..7f8c763d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.2", + "version": "0.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.2", + "version": "0.4.3", "license": "MIT", "devDependencies": { "@babel/core": "7.22.0", diff --git a/package.json b/package.json index d5c83991..d4e59048 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "git", "url": "https://github.com/mkkellogg/GaussianSplats3D" }, - "version": "0.4.2", + "version": "0.4.3", "description": "Three.js-based 3D Gaussian splat viewer", "module": "build/gaussian-splats-3d.module.js", "main": "build/gaussian-splats-3d.umd.cjs", diff --git a/src/DropInViewer.js b/src/DropInViewer.js index cbe15187..2c5573fa 100644 --- a/src/DropInViewer.js +++ b/src/DropInViewer.js @@ -20,11 +20,26 @@ export class DropInViewer extends THREE.Group { this.viewer = new Viewer(options); this.splatMesh = null; + this.updateSplatMesh(); this.callbackMesh = DropInViewer.createCallbackMesh(); this.add(this.callbackMesh); this.callbackMesh.onBeforeRender = DropInViewer.onBeforeRender.bind(this, this.viewer); + this.viewer.onSplatMeshChanged(() => { + this.updateSplatMesh(); + }); + + } + + updateSplatMesh() { + if (this.splatMesh !== this.viewer.splatMesh) { + if (this.splatMesh) { + this.remove(this.splatMesh); + } + this.splatMesh = this.viewer.splatMesh; + this.add(this.viewer.splatMesh); + } } /** @@ -85,8 +100,12 @@ export class DropInViewer extends THREE.Group { return this.viewer.getSplatScene(sceneIndex); } - removeSplatScene(index) { - return this.viewer.removeSplatScene(index); + removeSplatScene(index, showLoadingUI = true) { + return this.viewer.removeSplatScene(index, showLoadingUI); + } + + removeSplatScenes(indexes, showLoadingUI = true) { + return this.viewer.removeSplatScenes(indexes, showLoadingUI); } dispose() { @@ -94,13 +113,6 @@ export class DropInViewer extends THREE.Group { } static onBeforeRender(viewer, renderer, threeScene, camera) { - if (this.splatMesh !== this.viewer.splatMesh) { - if (this.splatMesh) { - this.remove(this.splatMesh); - } - this.splatMesh = this.viewer.splatMesh; - this.add(this.viewer.splatMesh); - } viewer.update(renderer, camera); } diff --git a/src/SplatRenderMode.js b/src/SplatRenderMode.js new file mode 100644 index 00000000..d4c72849 --- /dev/null +++ b/src/SplatRenderMode.js @@ -0,0 +1,4 @@ +export const SplatRenderMode = { + ThreeD: 0, + TwoD: 1 +}; diff --git a/src/Viewer.js b/src/Viewer.js index ae02cb63..69d903a3 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -23,6 +23,7 @@ import { LoaderStatus } from './loaders/LoaderStatus.js'; import { RenderMode } from './RenderMode.js'; import { LogLevel } from './LogLevel.js'; import { SceneRevealMode } from './SceneRevealMode.js'; +import { SplatRenderMode } from './SplatRenderMode.js'; const THREE_CAMERA_FOV = 50; const MINIMUM_DISTANCE_TO_NEW_FOCAL_POINT = .75; @@ -142,7 +143,7 @@ export class Viewer { this.logLevel = options.logLevel || LogLevel.None; // Degree of spherical harmonics to utilize in rendering splats (assuming the data is present in the splat scene). - // Valid values are 0 - 3. Default value is 0. + // Valid values are 0 - 2. Default value is 0. this.sphericalHarmonicsDegree = options.sphericalHarmonicsDegree || 0; // When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. @@ -180,6 +181,13 @@ export class Viewer { } } + // Tell the viewer how to render the splats + if (options.splatRenderMode === undefined || options.splatRenderMode === null) { + options.splatRenderMode = SplatRenderMode.ThreeD; + } + this.splatRenderMode = options.splatRenderMode; + + this.onSplatMeshChangedCallback = null; this.createSplatMesh(); this.controls = null; @@ -249,14 +257,17 @@ export class Viewer { this.initialized = false; this.disposing = false; this.disposed = false; + this.disposePromise = null; if (!this.dropInMode) this.init(); } createSplatMesh() { - this.splatMesh = new SplatMesh(this.dynamicScene, this.enableOptionalEffects, this.halfPrecisionCovariancesOnGPU, - this.devicePixelRatio, this.gpuAcceleratedSort, this.integerBasedSort, this.antialiased, - this.maxScreenSpaceSplatSize, this.logLevel, this.sphericalHarmonicsDegree); + this.splatMesh = new SplatMesh(this.splatRenderMode, this.dynamicScene, this.enableOptionalEffects, + this.halfPrecisionCovariancesOnGPU, this.devicePixelRatio, this.gpuAcceleratedSort, + this.integerBasedSort, this.antialiased, this.maxScreenSpaceSplatSize, this.logLevel, + this.sphericalHarmonicsDegree); this.splatMesh.frustumCulled = false; + if (this.onSplatMeshChangedCallback) this.onSplatMeshChangedCallback(); } init() { @@ -366,7 +377,7 @@ export class Viewer { this.perspectiveControls = new OrbitControls(this.camera, this.renderer.domElement); } } - for (let controls of [this.perspectiveControls, this.orthographicControls]) { + for (let controls of [this.orthographicControls, this.perspectiveControls,]) { if (controls) { controls.listenToKeyEvents(window); controls.rotateSpeed = 0.5; @@ -375,9 +386,11 @@ export class Viewer { controls.enableDamping = true; controls.dampingFactor = 0.05; controls.target.copy(this.initialCameraLookAt); + controls.update(); } } this.controls = this.camera.isOrthographicCamera ? this.orthographicControls : this.perspectiveControls; + this.controls.update(); } } @@ -411,6 +424,10 @@ export class Viewer { this.renderMode = renderMode; } + onSplatMeshChanged(callback) { + this.onSplatMeshChangedCallback = callback; + } + onKeyDown = function() { const forward = new THREE.Vector3(); @@ -1021,7 +1038,8 @@ export class Viewer { return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, splatAlphaRemovalThreshold, this.plyInMemoryCompressionLevel, this.sphericalHarmonicsDegree); } - return AbortablePromise.reject(new Error(`Viewer::downloadSplatSceneToSplatBuffer -> File format not supported: ${path}`)); + + throw new Error(`Viewer::downloadSplatSceneToSplatBuffer -> File format not supported: ${path}`); } static isProgressivelyLoadable(format) { @@ -1043,6 +1061,13 @@ export class Viewer { this.splatRenderReady = false; let splatProcessingTaskId = null; + const removeSplatProcessingTask = () => { + if (splatProcessingTaskId !== null) { + this.loadingSpinner.removeTask(splatProcessingTaskId); + splatProcessingTaskId = null; + } + }; + const finish = (buildResults, resolver) => { if (this.isDisposingOrDisposed()) return; @@ -1062,21 +1087,23 @@ export class Viewer { this.updateSplatSort(true); - if (enableRenderBeforeFirstSort) { + if (!this.sortWorker) { this.splatRenderReady = true; + removeSplatProcessingTask(); + resolver(); } else { - this.runAfterNextSort.push(() => { + if (enableRenderBeforeFirstSort) { this.splatRenderReady = true; + } else { + this.runAfterNextSort.push(() => { + this.splatRenderReady = true; + }); + } + this.runAfterNextSort.push(() => { + removeSplatProcessingTask(); + resolver(); }); } - - this.runAfterNextSort.push(() => { - if (splatProcessingTaskId !== null) { - this.loadingSpinner.removeTask(splatProcessingTaskId); - splatProcessingTaskId = null; - } - resolver(); - }); }; return new Promise((resolve) => { @@ -1245,7 +1272,11 @@ export class Viewer { this.sortRunning = false; } - removeSplatScene(index, showLoadingUI = true) { + removeSplatScene(indexToRemove, showLoadingUI = true) { + return this.removeSplatScenes([indexToRemove], showLoadingUI); + } + + removeSplatScenes(indexesToRemove, showLoadingUI = true) { if (this.isLoadingOrUnloading()) { throw new Error('Cannot remove splat scene while another load or unload is already in progress.'); } @@ -1294,7 +1325,14 @@ export class Viewer { const savedSceneOptions = []; const savedSceneTransformComponents = []; for (let i = 0; i < this.splatMesh.scenes.length; i++) { - if (i !== index) { + let shouldRemove = false; + for (let indexToRemove of indexesToRemove) { + if (indexToRemove === i) { + shouldRemove = true; + break; + } + } + if (!shouldRemove) { const scene = this.splatMesh.scenes[i]; savedSplatBuffers.push(scene.splatBuffer); savedSceneOptions.push(this.splatMesh.sceneOptions[i]); @@ -1307,6 +1345,7 @@ export class Viewer { } this.disposeSortWorker(); this.splatMesh.dispose(); + this.sceneRevealMode = SceneRevealMode.Instant; this.createSplatMesh(); this.addSplatBuffers(savedSplatBuffers, savedSceneOptions, true, false, true) .then(() => { @@ -1373,7 +1412,8 @@ export class Viewer { * Dispose of all resources held directly and indirectly by this viewer. */ async dispose() { - this.disposing = true; + if (this.isDisposingOrDisposed()) return this.disposePromise; + let waitPromises = []; let promisesToAbort = []; for (let promiseKey in this.splatSceneDownloadPromises) { @@ -1386,7 +1426,9 @@ export class Viewer { if (this.sortPromise) { waitPromises.push(this.sortPromise); } - const disposePromise = Promise.all(waitPromises).finally(() => { + + this.disposing = true; + this.disposePromise = Promise.all(waitPromises).finally(() => { this.stop(); if (this.controls) { this.controls.dispose(); @@ -1439,7 +1481,7 @@ export class Viewer { promisesToAbort.forEach((toAbort) => { toAbort.abort('Scene disposed'); }); - return disposePromise; + return this.disposePromise; } selfDrivenUpdate() { @@ -1756,7 +1798,10 @@ export class Viewer { return async function(force = false) { if (this.sortRunning) return; - if (this.splatMesh.getSplatCount() <= 0) return; + if (this.splatMesh.getSplatCount() <= 0) { + this.splatRenderCount = 0; + return; + } let angleDiff = 0; let positionDiff = 0; diff --git a/src/index.js b/src/index.js index ef561007..67f22108 100644 --- a/src/index.js +++ b/src/index.js @@ -17,6 +17,7 @@ import { WebXRMode } from './webxr/WebXRMode.js'; import { RenderMode } from './RenderMode.js'; import { LogLevel } from './LogLevel.js'; import { SceneRevealMode } from './SceneRevealMode.js'; +import { SplatRenderMode } from './SplatRenderMode.js'; export { PlyParser, @@ -37,5 +38,6 @@ export { WebXRMode, RenderMode, LogLevel, - SceneRevealMode + SceneRevealMode, + SplatRenderMode }; diff --git a/src/loaders/SplatBuffer.js b/src/loaders/SplatBuffer.js index e7f030fc..4ec7cf4c 100644 --- a/src/loaders/SplatBuffer.js +++ b/src/loaders/SplatBuffer.js @@ -6,6 +6,17 @@ import { Constants } from '../Constants.js'; const SphericalHarmonics8BitCompressionHalfRange = Constants.SphericalHarmonics8BitCompressionRange / 2.0; const toHalfFloat = THREE.DataUtils.toHalfFloat.bind(THREE.DataUtils); +const fromHalfFloat = THREE.DataUtils.fromHalfFloat.bind(THREE.DataUtils); + +const toUncompressedFloat = (f, compressionLevel, isSH = false) => { + if (compressionLevel === 0) { + return f; + } else if (compressionLevel === 1 || compressionLevel === 2 && !isSH) { + return THREE.DataUtils.fromHalfFloat(f); + } else if (compressionLevel === 2) { + return fromUint8(f); + } +}; const toUint8 = (v) => { v = clamp(v, -SphericalHarmonics8BitCompressionHalfRange, SphericalHarmonics8BitCompressionHalfRange); @@ -16,20 +27,12 @@ const fromUint8 = (v) => { return (v / 255) * Constants.SphericalHarmonics8BitCompressionRange - SphericalHarmonics8BitCompressionHalfRange; }; -const fromHalfFloat = THREE.DataUtils.fromHalfFloat.bind(THREE.DataUtils); - const fromHalfFloatToUint8 = (v) => { return toUint8(fromHalfFloat(v)); }; -const toUncompressedFloat = (f, compressionLevel, isSH = false) => { - if (compressionLevel === 0) { - return f; - } else if (compressionLevel === 1 || compressionLevel === 2 && !isSH) { - return THREE.DataUtils.fromHalfFloat(f); - } else if (compressionLevel === 2) { - return fromUint8(f); - } +const fromUint8ToHalfFloat = (v) => { + return toHalfFloat(fromUint8(v)); }; const dataViewFloatForCompressionLevel = (dataView, floatIndex, compressionLevel, isSH = false) => { @@ -42,6 +45,38 @@ const dataViewFloatForCompressionLevel = (dataView, floatIndex, compressionLevel } }; +const convertBetweenCompressionLevels = function() { + + const noop = (v) => v; + + return function(val, fromLevel, toLevel, isSH = false) { + if (fromLevel === toLevel) return val; + let outputConversionFunc = noop; + + if (fromLevel === 2 && isSH) { + if (toLevel === 1) outputConversionFunc = fromUint8ToHalfFloat; + else if (toLevel == 0) { + outputConversionFunc = fromUint8; + } + } else if (fromLevel === 2 || fromLevel === 1) { + if (toLevel === 0) outputConversionFunc = fromHalfFloat; + else if (toLevel == 2) { + if (!isSH) outputConversionFunc = noop; + else outputConversionFunc = fromHalfFloatToUint8; + } + } else if (fromLevel === 0) { + if (toLevel === 1) outputConversionFunc = toHalfFloat; + else if (toLevel == 2) { + if (!isSH) outputConversionFunc = toHalfFloat; + else outputConversionFunc = toUint8; + } + } + + return outputConversionFunc(val); + }; + +}(); + const copyBetweenBuffers = (srcBuffer, srcOffset, destBuffer, destOffset, byteCount = 0) => { const src = new Uint8Array(srcBuffer, srcOffset); const dest = new Uint8Array(destBuffer, destOffset); @@ -216,7 +251,7 @@ export class SplatBuffer { const scale = new THREE.Vector3(); const rotation = new THREE.Quaternion(); - return function(index, outScale, outRotation, transform) { + return function(index, outScale, outRotation, transform, scaleOverride) { const sectionIndex = this.globalSplatIndexToSectionMap[index]; const section = this.sections[sectionIndex]; const localSplatIndex = index - section.splatCountOffset; @@ -229,6 +264,11 @@ export class SplatBuffer { scale.set(toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 0, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 1, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 2, this.compressionLevel), this.compressionLevel)); + if (scaleOverride) { + if (scaleOverride.x !== undefined) scale.x = scaleOverride.x; + if (scaleOverride.y !== undefined) scale.y = scaleOverride.y; + if (scaleOverride.z !== undefined) scale.z = scaleOverride.z; + } rotation.set(toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 4, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 5, this.compressionLevel), this.compressionLevel), @@ -303,6 +343,97 @@ export class SplatBuffer { } } + fillSplatScaleRotationArray = function() { + + const scaleMatrix = new THREE.Matrix4(); + const rotationMatrix = new THREE.Matrix4(); + const tempMatrix = new THREE.Matrix4(); + const scale = new THREE.Vector3(); + const rotation = new THREE.Quaternion(); + const tempPosition = new THREE.Vector3(); + + const ensurePositiveW = (quaternion) => { + const flip = quaternion.w < 0 ? -1 : 1; + quaternion.x *= flip; + quaternion.y *= flip; + quaternion.z *= flip; + quaternion.w *= flip; + }; + + return function(outScaleArray, outRotationArray, transform, srcFrom, srcTo, destFrom, + desiredOutputCompressionLevel, scaleOverride) { + const splatCount = this.splatCount; + + srcFrom = srcFrom || 0; + srcTo = srcTo || splatCount - 1; + if (destFrom === undefined) destFrom = srcFrom; + + const outputConversion = (value, srcCompressionLevel) => { + if (srcCompressionLevel === undefined) srcCompressionLevel = this.compressionLevel; + return convertBetweenCompressionLevels(value, srcCompressionLevel, desiredOutputCompressionLevel); + }; + + for (let i = srcFrom; i <= srcTo; i++) { + const sectionIndex = this.globalSplatIndexToSectionMap[i]; + const section = this.sections[sectionIndex]; + const localSplatIndex = i - section.splatCountOffset; + + const srcSplatScalesBase = section.bytesPerSplat * localSplatIndex + + SplatBuffer.CompressionLevels[this.compressionLevel].ScaleOffsetBytes; + + const scaleDestBase = (i - srcFrom + destFrom) * SplatBuffer.ScaleComponentCount; + const rotationDestBase = (i - srcFrom + destFrom) * SplatBuffer.RotationComponentCount; + const dataView = new DataView(this.bufferData, section.dataBase + srcSplatScalesBase); + + const srcScaleX = (scaleOverride && scaleOverride.x !== undefined) ? scaleOverride.x : + dataViewFloatForCompressionLevel(dataView, 0, this.compressionLevel); + const srcScaleY = (scaleOverride && scaleOverride.y !== undefined) ? scaleOverride.y : + dataViewFloatForCompressionLevel(dataView, 1, this.compressionLevel); + const srcScaleZ = (scaleOverride && scaleOverride.z !== undefined) ? scaleOverride.z : + dataViewFloatForCompressionLevel(dataView, 2, this.compressionLevel); + + const srcRotationW = dataViewFloatForCompressionLevel(dataView, 3, this.compressionLevel); + const srcRotationX = dataViewFloatForCompressionLevel(dataView, 4, this.compressionLevel); + const srcRotationY = dataViewFloatForCompressionLevel(dataView, 5, this.compressionLevel); + const srcRotationZ = dataViewFloatForCompressionLevel(dataView, 6, this.compressionLevel); + + scale.set(toUncompressedFloat(srcScaleX, this.compressionLevel), + toUncompressedFloat(srcScaleY, this.compressionLevel), + toUncompressedFloat(srcScaleZ, this.compressionLevel)); + + rotation.set(toUncompressedFloat(srcRotationX, this.compressionLevel), + toUncompressedFloat(srcRotationY, this.compressionLevel), + toUncompressedFloat(srcRotationZ, this.compressionLevel), + toUncompressedFloat(srcRotationW, this.compressionLevel)).normalize(); + + if (transform) { + tempPosition.set(0, 0, 0); + scaleMatrix.makeScale(scale.x, scale.y, scale.z); + rotationMatrix.makeRotationFromQuaternion(rotation); + tempMatrix.identity().premultiply(scaleMatrix).premultiply(rotationMatrix); + tempMatrix.premultiply(transform); + tempMatrix.decompose(tempPosition, rotation, scale); + rotation.normalize(); + } + + ensurePositiveW(rotation); + + if (outScaleArray) { + outScaleArray[scaleDestBase] = outputConversion(scale.x, 0); + outScaleArray[scaleDestBase + 1] = outputConversion(scale.y, 0); + outScaleArray[scaleDestBase + 2] = outputConversion(scale.z, 0); + } + + if (outRotationArray) { + outRotationArray[rotationDestBase] = outputConversion(rotation.x, 0); + outRotationArray[rotationDestBase + 1] = outputConversion(rotation.y, 0); + outRotationArray[rotationDestBase + 2] = outputConversion(rotation.z, 0); + outRotationArray[rotationDestBase + 3] = outputConversion(rotation.w, 0); + } + } + }; + }(); + static computeCovariance = function() { const tempMatrix4 = new THREE.Matrix4(); @@ -375,6 +506,7 @@ export class SplatBuffer { scale.set(toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 0, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 1, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 2, this.compressionLevel), this.compressionLevel)); + rotation.set(toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 4, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 5, this.compressionLevel), this.compressionLevel), toUncompressedFloat(dataViewFloatForCompressionLevel(dataView, 6, this.compressionLevel), this.compressionLevel), @@ -421,6 +553,11 @@ export class SplatBuffer { } const tempMatrix3 = new THREE.Matrix3(); + const tempMatrix4 = new THREE.Matrix4(); + + const tempTranslation = new THREE.Vector3(); + const tempScale = new THREE.Vector3(); + const tempRotation = new THREE.Quaternion(); const sh11 = []; const sh12 = []; @@ -486,7 +623,11 @@ export class SplatBuffer { if (destFrom === undefined) destFrom = srcFrom; if (transform && outSphericalHarmonicsDegree >= 1) { - tempMatrix3.setFromMatrix4(transform); + tempMatrix4.copy(transform); + tempMatrix4.decompose(tempTranslation, tempRotation, tempScale); + tempRotation.normalize(); + tempMatrix4.makeRotationFromQuaternion(tempRotation); + tempMatrix3.setFromMatrix4(tempMatrix4); set3(sh11, tempMatrix3.elements[4], -tempMatrix3.elements[7], tempMatrix3.elements[1]); set3(sh12, -tempMatrix3.elements[5], tempMatrix3.elements[8], -tempMatrix3.elements[2]); set3(sh13, tempMatrix3.elements[3], -tempMatrix3.elements[6], tempMatrix3.elements[0]); @@ -931,9 +1072,11 @@ export class SplatBuffer { } if (targetSplat[OFFSET_SCALE0] !== undefined) { - tempScale.set(targetSplat[OFFSET_SCALE0], targetSplat[OFFSET_SCALE1], targetSplat[OFFSET_SCALE2]); + tempScale.set(targetSplat[OFFSET_SCALE0] || 0, + targetSplat[OFFSET_SCALE1] || 0, + targetSplat[OFFSET_SCALE2] || 0); } else { - tempScale.set(0.01, 0.01, 0.01); + tempScale.set(0, 0, 0); } if (compressionLevel === 0) { diff --git a/src/loaders/ply/INRIAV1PlyParser.js b/src/loaders/ply/INRIAV1PlyParser.js index 4c0f2d60..4fbaed82 100644 --- a/src/loaders/ply/INRIAV1PlyParser.js +++ b/src/loaders/ply/INRIAV1PlyParser.js @@ -4,18 +4,14 @@ import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; import { SplatBuffer } from '../SplatBuffer.js'; import { PlyParserUtils } from './PlyParserUtils.js'; -const FieldNamesToRead = ['scale_0', 'scale_1', 'scale_2', 'rot_0', 'rot_1', 'rot_2', 'rot_3', - 'x', 'y', 'z', 'f_dc_0', 'f_dc_1', 'f_dc_2', 'opacity', 'red', 'green', 'blue', - 'f_rest_0', 'f_rest_1', 'f_rest_2', 'f_rest_15', 'f_rest_16', 'f_rest_17', 'f_rest_30', 'f_rest_31', 'f_rest_32', - 'f_rest_3', 'f_rest_4', 'f_rest_5', 'f_rest_6', 'f_rest_7', - 'f_rest_18', 'f_rest_19', 'f_rest_20', 'f_rest_21', 'f_rest_22', - 'f_rest_33', 'f_rest_34', 'f_rest_35', 'f_rest_36', 'f_rest_37']; +const BaseFieldNamesToRead = ['scale_0', 'scale_1', 'scale_2', 'rot_0', 'rot_1', 'rot_2', 'rot_3', 'x', 'y', 'z', + 'f_dc_0', 'f_dc_1', 'f_dc_2', 'opacity', 'red', 'green', 'blue', 'f_rest_0']; -const FieldsToReadIndexes = FieldNamesToRead.map((e, i) => i); +const BaseFieldsToReadIndexes = BaseFieldNamesToRead.map((e, i) => i); const [ SCALE_0, SCALE_1, SCALE_2, ROT_0, ROT_1, ROT_2, ROT_3, X, Y, Z, F_DC_0, F_DC_1, F_DC_2, OPACITY, RED, GREEN, BLUE, F_REST_0 - ] = FieldsToReadIndexes; + ] = BaseFieldsToReadIndexes; export class INRIAV1PlyParser { @@ -24,13 +20,35 @@ export class INRIAV1PlyParser { } decodeHeaderLines(headerLines) { - const fieldNameIdMap = FieldsToReadIndexes.reduce((acc, element) => { - acc[FieldNamesToRead[element]] = element; + + let shLineCount = 0; + headerLines.forEach((line) => { + if (line.includes('f_rest_')) shLineCount++; + }); + + let shFieldsToReadCount = 0; + if (shLineCount >= 45) { + shFieldsToReadCount = 45; + } else if (shLineCount >= 24) { + shFieldsToReadCount = 24; + } else if (shLineCount >= 9) { + shFieldsToReadCount = 9; + } + + const shFieldIndexesToMap = Array.from(Array(Math.max(shFieldsToReadCount - 1, 0))); + let shRemainingFieldNamesToRead = shFieldIndexesToMap.map((element, index) => `f_rest_${index + 1}`); + + const fieldNamesToRead = [...BaseFieldNamesToRead, ...shRemainingFieldNamesToRead]; + const fieldsToReadIndexes = fieldNamesToRead.map((e, i) => i); + + const fieldNameIdMap = fieldsToReadIndexes.reduce((acc, element) => { + acc[fieldNamesToRead[element]] = element; return acc; }, {}); const header = this.plyParserutils.decodeSectionHeader(headerLines, fieldNameIdMap, 0); header.splatCount = header.vertexCount; header.bytesPerSplat = header.bytesPerVertex; + header.fieldsToReadIndexes = fieldsToReadIndexes; return header; } @@ -52,7 +70,7 @@ export class INRIAV1PlyParser { } parseToUncompressedSplatBufferSection(header, fromSplat, toSplat, splatData, splatDataOffset, - toBuffer, toOffset, outSphericalHarmonicsDegree = 0) { + toBuffer, toOffset, outSphericalHarmonicsDegree = 0) { outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); const outBytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree].BytesPerSplat; @@ -173,7 +191,7 @@ export class INRIAV1PlyParser { }(); static readSplat(splatData, header, row, dataOffset, rawSplat) { - return PlyParserUtils.readVertex(splatData, header, row, dataOffset, FieldsToReadIndexes, rawSplat, true); + return PlyParserUtils.readVertex(splatData, header, row, dataOffset, header.fieldsToReadIndexes, rawSplat, true); } parseToUncompressedSplatArray(plyBuffer, outSphericalHarmonicsDegree = 0) { diff --git a/src/raycaster/Raycaster.js b/src/raycaster/Raycaster.js index a3652a26..910a1e97 100644 --- a/src/raycaster/Raycaster.js +++ b/src/raycaster/Raycaster.js @@ -1,6 +1,7 @@ import * as THREE from 'three'; import { Ray } from './Ray.js'; import { Hit } from './Hit.js'; +import { SplatRenderMode } from '../SplatRenderMode.js'; export class Raycaster { @@ -116,12 +117,19 @@ export class Raycaster { splatTree.splatMesh.getSplatCenter(splatGlobalIndex, tempCenter); splatTree.splatMesh.getSplatScaleAndRotation(splatGlobalIndex, tempScale, tempRotation); - if (tempScale.x <= scaleEpsilon || tempScale.y <= scaleEpsilon || tempScale.z <= scaleEpsilon) { + if (tempScale.x <= scaleEpsilon || tempScale.y <= scaleEpsilon || + splatTree.splatMesh.splatRenderMode === SplatRenderMode.ThreeD && tempScale.z <= scaleEpsilon) { continue; } if (!this.raycastAgainstTrueSplatEllipsoid) { - const radius = (tempScale.x + tempScale.y + tempScale.z) / 3; + let radius = (tempScale.x + tempScale.y); + let componentCount = 2; + if (splatTree.splatMesh.splatRenderMode === SplatRenderMode.ThreeD) { + radius += tempScale.z; + componentCount = 3; + } + radius = radius / componentCount; if (ray.intersectSphere(tempCenter, radius, tempHit)) { const hitClone = tempHit.clone(); hitClone.splatIndex = splatGlobalIndex; diff --git a/src/splatmesh/SplatMaterial.js b/src/splatmesh/SplatMaterial.js index a724a562..da3aaab0 100644 --- a/src/splatmesh/SplatMaterial.js +++ b/src/splatmesh/SplatMaterial.js @@ -3,491 +3,345 @@ import { Constants } from '../Constants.js'; export class SplatMaterial { - /** - * Build the Three.js material that is used to render the splats. - * @param {number} dynamicMode If true, it means the scene geometry represented by this splat mesh is not stationary or - * that the splat count might change - * @param {boolean} enableOptionalEffects When true, allows for usage of extra properties and attributes in the shader for effects - * such as opacity adjustment. Default is false for performance reasons. - * @param {boolean} antialiased If true, calculate compensation factor to deal with gaussians being rendered at a significantly - * different resolution than that of their training - * @param {number} maxScreenSpaceSplatSize The maximum clip space splat size - * @param {number} splatScale Value by which all splats are scaled in screen-space (default is 1.0) - * @param {number} pointCloudModeEnabled Render all splats as screen-space circles - * @param {number} maxSphericalHarmonicsDegree Degree of spherical harmonics to utilize in rendering splats - * @return {THREE.ShaderMaterial} - */ - static build(dynamicMode = false, enableOptionalEffects = false, antialiased = false, maxScreenSpaceSplatSize = 2048, - splatScale = 1.0, pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0) { - // Contains the code to project 3D covariance to 2D and from there calculate the quad (using the eigen vectors of the - // 2D covariance) that is ultimately rasterized + static buildVertexShaderBase(dynamicMode = false, enableOptionalEffects = false, maxSphericalHarmonicsDegree = 0, customVars = '') { let vertexShaderSource = ` - precision highp float; - #include + precision highp float; + #include + + attribute uint splatIndex; + uniform highp usampler2D centersColorsTexture; + uniform highp sampler2D sphericalHarmonicsTexture; + uniform highp sampler2D sphericalHarmonicsTextureR; + uniform highp sampler2D sphericalHarmonicsTextureG; + uniform highp sampler2D sphericalHarmonicsTextureB; + `; + + if (enableOptionalEffects || dynamicMode) { + vertexShaderSource += ` + uniform highp usampler2D sceneIndexesTexture; + uniform vec2 sceneIndexesTextureSize; + `; + } + + if (enableOptionalEffects) { + vertexShaderSource += ` + uniform float sceneOpacity[${Constants.MaxScenes}]; + uniform int sceneVisibility[${Constants.MaxScenes}]; + `; + } + + if (dynamicMode) { + vertexShaderSource += ` + uniform highp mat4 transforms[${Constants.MaxScenes}]; + `; + } + + vertexShaderSource += ` + ${customVars} + uniform vec2 focal; + uniform float orthoZoom; + uniform int orthographicMode; + uniform int pointCloudModeEnabled; + uniform float inverseFocalAdjustment; + uniform vec2 viewport; + uniform vec2 basisViewport; + uniform vec2 centersColorsTextureSize; + uniform int sphericalHarmonicsDegree; + uniform vec2 sphericalHarmonicsTextureSize; + uniform int sphericalHarmonics8BitMode; + uniform int sphericalHarmonicsMultiTextureMode; + uniform float visibleRegionRadius; + uniform float visibleRegionFadeStartRadius; + uniform float firstRenderTime; + uniform float currentTime; + uniform int fadeInComplete; + uniform vec3 sceneCenter; + uniform float splatScale; + + varying vec4 vColor; + varying vec2 vUv; + varying vec2 vPosition; + + mat3 quaternionToRotationMatrix(float x, float y, float z, float w) { + float s = 1.0 / sqrt(w * w + x * x + y * y + z * z); + + return mat3( + 1. - 2. * (y * y + z * z), + 2. * (x * y + w * z), + 2. * (x * z - w * y), + 2. * (x * y - w * z), + 1. - 2. * (x * x + z * z), + 2. * (y * z + w * x), + 2. * (x * z + w * y), + 2. * (y * z - w * x), + 1. - 2. * (x * x + y * y) + ); + } - attribute uint splatIndex; + const float sqrt8 = sqrt(8.0); + const float minAlpha = 1.0 / 255.0; + + const vec4 encodeNorm4 = vec4(1.0 / 255.0, 1.0 / 255.0, 1.0 / 255.0, 1.0 / 255.0); + const uvec4 mask4 = uvec4(uint(0x000000FF), uint(0x0000FF00), uint(0x00FF0000), uint(0xFF000000)); + const uvec4 shift4 = uvec4(0, 8, 16, 24); + vec4 uintToRGBAVec (uint u) { + uvec4 urgba = mask4 & u; + urgba = urgba >> shift4; + vec4 rgba = vec4(urgba) * encodeNorm4; + return rgba; + } - uniform highp sampler2D covariancesTexture; - uniform highp usampler2D centersColorsTexture; - uniform highp sampler2D sphericalHarmonicsTexture; - uniform highp sampler2D sphericalHarmonicsTextureR; - uniform highp sampler2D sphericalHarmonicsTextureG; - uniform highp sampler2D sphericalHarmonicsTextureB;`; + vec2 getDataUV(in int stride, in int offset, in vec2 dimensions) { + vec2 samplerUV = vec2(0.0, 0.0); + float d = float(splatIndex * uint(stride) + uint(offset)) / dimensions.x; + samplerUV.y = float(floor(d)) / dimensions.y; + samplerUV.x = fract(d); + return samplerUV; + } + + vec2 getDataUVF(in uint sIndex, in float stride, in uint offset, in vec2 dimensions) { + vec2 samplerUV = vec2(0.0, 0.0); + float d = float(uint(float(sIndex) * stride) + offset) / dimensions.x; + samplerUV.y = float(floor(d)) / dimensions.y; + samplerUV.x = fract(d); + return samplerUV; + } - if (enableOptionalEffects || dynamicMode) { + const float SH_C1 = 0.4886025119029199f; + const float[5] SH_C2 = float[](1.0925484, -1.0925484, 0.3153916, -1.0925484, 0.5462742); + + const float SphericalHarmonics8BitCompressionRange = ${Constants.SphericalHarmonics8BitCompressionRange.toFixed(1)}; + const float SphericalHarmonics8BitCompressionHalfRange = SphericalHarmonics8BitCompressionRange / 2.0; + const vec3 vec8BitSHShift = vec3(SphericalHarmonics8BitCompressionHalfRange); + + void main () { + + uint oddOffset = splatIndex & uint(0x00000001); + uint doubleOddOffset = oddOffset * uint(2); + bool isEven = oddOffset == uint(0); + uint nearestEvenIndex = splatIndex - oddOffset; + float fOddOffset = float(oddOffset); + + uvec4 sampledCenterColor = texture(centersColorsTexture, getDataUV(1, 0, centersColorsTextureSize)); + vec3 splatCenter = uintBitsToFloat(uvec3(sampledCenterColor.gba));`; + + if (dynamicMode || enableOptionalEffects) { vertexShaderSource += ` - uniform highp usampler2D sceneIndexesTexture; - uniform vec2 sceneIndexesTextureSize; + uint sceneIndex = texture(sceneIndexesTexture, getDataUV(1, 0, sceneIndexesTextureSize)).r; `; } if (enableOptionalEffects) { vertexShaderSource += ` - uniform float sceneOpacity[${Constants.MaxScenes}]; - uniform int sceneVisibility[${Constants.MaxScenes}]; + float splatOpacityFromScene = sceneOpacity[sceneIndex]; + int sceneVisible = sceneVisibility[sceneIndex]; + if (splatOpacityFromScene <= 0.01 || sceneVisible == 0) { + gl_Position = vec4(0.0, 0.0, 2.0, 1.0); + return; + } `; } if (dynamicMode) { vertexShaderSource += ` - uniform highp mat4 transforms[${Constants.MaxScenes}]; + mat4 transform = transforms[sceneIndex]; + mat4 transformModelViewMatrix = modelViewMatrix * transform; `; + } else { + vertexShaderSource += `mat4 transformModelViewMatrix = modelViewMatrix;`; } vertexShaderSource += ` - uniform vec2 focal; - uniform float orthoZoom; - uniform int orthographicMode; - uniform int pointCloudModeEnabled; - uniform float inverseFocalAdjustment; - uniform vec2 viewport; - uniform vec2 basisViewport; - uniform vec2 covariancesTextureSize; - uniform vec2 centersColorsTextureSize; - uniform int sphericalHarmonicsDegree; - uniform vec2 sphericalHarmonicsTextureSize; - uniform int sphericalHarmonics8BitMode; - uniform int sphericalHarmonicsMultiTextureMode; - uniform float visibleRegionRadius; - uniform float visibleRegionFadeStartRadius; - uniform float firstRenderTime; - uniform float currentTime; - uniform int fadeInComplete; - uniform vec3 sceneCenter; - uniform float splatScale; - - varying vec4 vColor; - varying vec2 vUv; - - varying vec2 vPosition; - - const float sqrt8 = sqrt(8.0); - const float minAlpha = 1.0 / 255.0; - - const vec4 encodeNorm4 = vec4(1.0 / 255.0, 1.0 / 255.0, 1.0 / 255.0, 1.0 / 255.0); - const uvec4 mask4 = uvec4(uint(0x000000FF), uint(0x0000FF00), uint(0x00FF0000), uint(0xFF000000)); - const uvec4 shift4 = uvec4(0, 8, 16, 24); - vec4 uintToRGBAVec (uint u) { - uvec4 urgba = mask4 & u; - urgba = urgba >> shift4; - vec4 rgba = vec4(urgba) * encodeNorm4; - return rgba; - } + vec4 viewCenter = transformModelViewMatrix * vec4(splatCenter, 1.0); - vec2 getDataUV(in int stride, in int offset, in vec2 dimensions) { - vec2 samplerUV = vec2(0.0, 0.0); - float d = float(splatIndex * uint(stride) + uint(offset)) / dimensions.x; - samplerUV.y = float(floor(d)) / dimensions.y; - samplerUV.x = fract(d); - return samplerUV; - } + vec4 clipCenter = projectionMatrix * viewCenter; - vec2 getDataUVF(in uint sIndex, in float stride, in uint offset, in vec2 dimensions) { - vec2 samplerUV = vec2(0.0, 0.0); - float d = float(uint(float(sIndex) * stride) + offset) / dimensions.x; - samplerUV.y = float(floor(d)) / dimensions.y; - samplerUV.x = fract(d); - return samplerUV; + float clip = 1.2 * clipCenter.w; + if (clipCenter.z < -clip || clipCenter.x < -clip || clipCenter.x > clip || clipCenter.y < -clip || clipCenter.y > clip) { + gl_Position = vec4(0.0, 0.0, 2.0, 1.0); + return; } - const float SH_C1 = 0.4886025119029199f; - const float[5] SH_C2 = float[](1.0925484, -1.0925484, 0.3153916, -1.0925484, 0.5462742); - - const float SphericalHarmonics8BitCompressionRange = ${Constants.SphericalHarmonics8BitCompressionRange.toFixed(1)}; - const float SphericalHarmonics8BitCompressionHalfRange = SphericalHarmonics8BitCompressionRange / 2.0; - const vec3 vec8BitSHShift = vec3(SphericalHarmonics8BitCompressionHalfRange); + vec3 ndcCenter = clipCenter.xyz / clipCenter.w; - void main () { + vPosition = position.xy; + vColor = uintToRGBAVec(sampledCenterColor.r); + `; - uint oddOffset = splatIndex & uint(0x00000001); - uint doubleOddOffset = oddOffset * uint(2); - bool isEven = oddOffset == uint(0); - uint nearestEvenIndex = splatIndex - oddOffset; - float fOddOffset = float(oddOffset); + if (maxSphericalHarmonicsDegree >= 1) { - uvec4 sampledCenterColor = texture(centersColorsTexture, getDataUV(1, 0, centersColorsTextureSize)); - vec3 splatCenter = uintBitsToFloat(uvec3(sampledCenterColor.gba));`; + vertexShaderSource += ` + if (sphericalHarmonicsDegree >= 1) { + `; - if (dynamicMode || enableOptionalEffects) { + if (dynamicMode) { vertexShaderSource += ` - uint sceneIndex = texture(sceneIndexesTexture, getDataUV(1, 0, sceneIndexesTextureSize)).r; + mat4 mTransform = modelMatrix * transform; + vec3 worldViewDir = normalize(splatCenter - vec3(inverse(mTransform) * vec4(cameraPosition, 1.0))); `; - } - - if (enableOptionalEffects) { + } else { vertexShaderSource += ` - float splatOpacityFromScene = sceneOpacity[sceneIndex]; - int sceneVisible = sceneVisibility[sceneIndex]; - if (splatOpacityFromScene <= 0.01 || sceneVisible == 0) { - gl_Position = vec4(0.0, 0.0, 2.0, 1.0); - return; - } + vec3 worldViewDir = normalize(splatCenter - cameraPosition); `; } - if (dynamicMode) { + vertexShaderSource += ` + vec3 sh1; + vec3 sh2; + vec3 sh3; + `; + + if (maxSphericalHarmonicsDegree >= 2) { vertexShaderSource += ` - mat4 transform = transforms[sceneIndex]; - mat4 transformModelViewMatrix = modelViewMatrix * transform; + vec4 sampledSH0123; + vec4 sampledSH4567; + vec4 sampledSH891011; + + vec4 sampledSH0123R; + vec4 sampledSH0123G; + vec4 sampledSH0123B; + + if (sphericalHarmonicsMultiTextureMode == 0) { + sampledSH0123 = texture(sphericalHarmonicsTexture, getDataUV(6, 0, sphericalHarmonicsTextureSize)); + sampledSH4567 = texture(sphericalHarmonicsTexture, getDataUV(6, 1, sphericalHarmonicsTextureSize)); + sampledSH891011 = texture(sphericalHarmonicsTexture, getDataUV(6, 2, sphericalHarmonicsTextureSize)); + sh1 = sampledSH0123.rgb; + sh2 = vec3(sampledSH0123.a, sampledSH4567.rg); + sh3 = vec3(sampledSH4567.ba, sampledSH891011.r); + } else { + sampledSH0123R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sampledSH0123G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sampledSH0123B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sh1 = vec3(sampledSH0123R.rgb); + sh2 = vec3(sampledSH0123G.rgb); + sh3 = vec3(sampledSH0123B.rgb); + } `; } else { - vertexShaderSource += `mat4 transformModelViewMatrix = modelViewMatrix;`; + vertexShaderSource += ` + if (sphericalHarmonicsMultiTextureMode == 0) { + vec2 shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset, sphericalHarmonicsTextureSize); + vec4 sampledSH0123 = texture(sphericalHarmonicsTexture, shUV); + shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset + uint(1), sphericalHarmonicsTextureSize); + vec4 sampledSH4567 = texture(sphericalHarmonicsTexture, shUV); + shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset + uint(2), sphericalHarmonicsTextureSize); + vec4 sampledSH891011 = texture(sphericalHarmonicsTexture, shUV); + sh1 = vec3(sampledSH0123.rgb) * (1.0 - fOddOffset) + vec3(sampledSH0123.ba, sampledSH4567.r) * fOddOffset; + sh2 = vec3(sampledSH0123.a, sampledSH4567.rg) * (1.0 - fOddOffset) + vec3(sampledSH4567.gba) * fOddOffset; + sh3 = vec3(sampledSH4567.ba, sampledSH891011.r) * (1.0 - fOddOffset) + vec3(sampledSH891011.rgb) * fOddOffset; + } else { + vec2 sampledSH01R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)).rg; + vec2 sampledSH23R = texture(sphericalHarmonicsTextureR, getDataUV(2, 1, sphericalHarmonicsTextureSize)).rg; + vec2 sampledSH01G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)).rg; + vec2 sampledSH23G = texture(sphericalHarmonicsTextureG, getDataUV(2, 1, sphericalHarmonicsTextureSize)).rg; + vec2 sampledSH01B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)).rg; + vec2 sampledSH23B = texture(sphericalHarmonicsTextureB, getDataUV(2, 1, sphericalHarmonicsTextureSize)).rg; + sh1 = vec3(sampledSH01R.rg, sampledSH23R.r); + sh2 = vec3(sampledSH01G.rg, sampledSH23G.r); + sh3 = vec3(sampledSH01B.rg, sampledSH23B.r); + } + `; } vertexShaderSource += ` - vec4 viewCenter = transformModelViewMatrix * vec4(splatCenter, 1.0); - - vec4 clipCenter = projectionMatrix * viewCenter; - - float clip = 1.2 * clipCenter.w; - if (clipCenter.z < -clip || clipCenter.x < -clip || clipCenter.x > clip || clipCenter.y < -clip || clipCenter.y > clip) { - gl_Position = vec4(0.0, 0.0, 2.0, 1.0); - return; - } - - vPosition = position.xy; - vColor = uintToRGBAVec(sampledCenterColor.r); + if (sphericalHarmonics8BitMode == 1) { + sh1 = sh1 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh2 = sh2 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh3 = sh3 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + } + float x = worldViewDir.x; + float y = worldViewDir.y; + float z = worldViewDir.z; + vColor.rgb += SH_C1 * (-sh1 * y + sh2 * z - sh3 * x); `; - if (maxSphericalHarmonicsDegree >= 1) { - - vertexShaderSource += ` - if (sphericalHarmonicsDegree >= 1) { - `; - - if (dynamicMode) { - vertexShaderSource += ` - mat4 mTransform = modelMatrix * transform; - vec3 worldViewDir = normalize(splatCenter - vec3(inverse(mTransform) * vec4(cameraPosition, 1.0))); - `; - } else { - vertexShaderSource += ` - vec3 worldViewDir = normalize(splatCenter - cameraPosition); - `; - } + if (maxSphericalHarmonicsDegree >= 2) { vertexShaderSource += ` - vec3 sh1; - vec3 sh2; - vec3 sh3; - `; - - if (maxSphericalHarmonicsDegree >= 2) { - vertexShaderSource += ` - vec4 sampledSH0123; - vec4 sampledSH4567; - vec4 sampledSH891011; + if (sphericalHarmonicsDegree >= 2) { + float xx = x * x; + float yy = y * y; + float zz = z * z; + float xy = x * y; + float yz = y * z; + float xz = x * z; + + vec3 sh4; + vec3 sh5; + vec3 sh6; + vec3 sh7; + vec3 sh8; - vec4 sampledSH0123R; - vec4 sampledSH0123G; - vec4 sampledSH0123B; - if (sphericalHarmonicsMultiTextureMode == 0) { - sampledSH0123 = texture(sphericalHarmonicsTexture, getDataUV(6, 0, sphericalHarmonicsTextureSize)); - sampledSH4567 = texture(sphericalHarmonicsTexture, getDataUV(6, 1, sphericalHarmonicsTextureSize)); - sampledSH891011 = texture(sphericalHarmonicsTexture, getDataUV(6, 2, sphericalHarmonicsTextureSize)); - sh1 = sampledSH0123.rgb; - sh2 = vec3(sampledSH0123.a, sampledSH4567.rg); - sh3 = vec3(sampledSH4567.ba, sampledSH891011.r); + vec4 sampledSH12131415 = texture(sphericalHarmonicsTexture, getDataUV(6, 3, sphericalHarmonicsTextureSize)); + vec4 sampledSH16171819 = texture(sphericalHarmonicsTexture, getDataUV(6, 4, sphericalHarmonicsTextureSize)); + vec4 sampledSH20212223 = texture(sphericalHarmonicsTexture, getDataUV(6, 5, sphericalHarmonicsTextureSize)); + sh4 = sampledSH891011.gba; + sh5 = sampledSH12131415.rgb; + sh6 = vec3(sampledSH12131415.a, sampledSH16171819.rg); + sh7 = vec3(sampledSH16171819.ba, sampledSH20212223.r); + sh8 = sampledSH20212223.gba; } else { - sampledSH0123R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sampledSH0123G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sampledSH0123B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sh1 = vec3(sampledSH0123R.rgb); - sh2 = vec3(sampledSH0123G.rgb); - sh3 = vec3(sampledSH0123B.rgb); + vec4 sampledSH4567R = texture(sphericalHarmonicsTextureR, getDataUV(2, 1, sphericalHarmonicsTextureSize)); + vec4 sampledSH4567G = texture(sphericalHarmonicsTextureG, getDataUV(2, 1, sphericalHarmonicsTextureSize)); + vec4 sampledSH4567B = texture(sphericalHarmonicsTextureB, getDataUV(2, 1, sphericalHarmonicsTextureSize)); + sh4 = vec3(sampledSH0123R.a, sampledSH4567R.rg); + sh5 = vec3(sampledSH4567R.ba, sampledSH0123G.a); + sh6 = vec3(sampledSH4567G.rgb); + sh7 = vec3(sampledSH4567G.a, sampledSH0123B.a, sampledSH4567B.r); + sh8 = vec3(sampledSH4567B.gba); } - `; - } else { - vertexShaderSource += ` - if (sphericalHarmonicsMultiTextureMode == 0) { - vec2 shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset, sphericalHarmonicsTextureSize); - vec4 sampledSH0123 = texture(sphericalHarmonicsTexture, shUV); - shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset + uint(1), sphericalHarmonicsTextureSize); - vec4 sampledSH4567 = texture(sphericalHarmonicsTexture, shUV); - shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset + uint(2), sphericalHarmonicsTextureSize); - vec4 sampledSH891011 = texture(sphericalHarmonicsTexture, shUV); - sh1 = vec3(sampledSH0123.rgb) * (1.0 - fOddOffset) + vec3(sampledSH0123.ba, sampledSH4567.r) * fOddOffset; - sh2 = vec3(sampledSH0123.a, sampledSH4567.rg) * (1.0 - fOddOffset) + vec3(sampledSH4567.gba) * fOddOffset; - sh3 = vec3(sampledSH4567.ba, sampledSH891011.r) * (1.0 - fOddOffset) + vec3(sampledSH891011.rgb) * fOddOffset; - } else { - vec2 sampledSH01R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)).rg; - vec2 sampledSH23R = texture(sphericalHarmonicsTextureR, getDataUV(2, 1, sphericalHarmonicsTextureSize)).rg; - vec2 sampledSH01G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)).rg; - vec2 sampledSH23G = texture(sphericalHarmonicsTextureG, getDataUV(2, 1, sphericalHarmonicsTextureSize)).rg; - vec2 sampledSH01B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)).rg; - vec2 sampledSH23B = texture(sphericalHarmonicsTextureB, getDataUV(2, 1, sphericalHarmonicsTextureSize)).rg; - sh1 = vec3(sampledSH01R.rg, sampledSH23R.r); - sh2 = vec3(sampledSH01G.rg, sampledSH23G.r); - sh3 = vec3(sampledSH01B.rg, sampledSH23B.r); - } - `; - } - vertexShaderSource += ` if (sphericalHarmonics8BitMode == 1) { - sh1 = sh1 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh2 = sh2 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh3 = sh3 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - } - float x = worldViewDir.x; - float y = worldViewDir.y; - float z = worldViewDir.z; - vColor.rgb += SH_C1 * (-sh1 * y + sh2 * z - sh3 * x); - `; - - if (maxSphericalHarmonicsDegree >= 2) { - - vertexShaderSource += ` - if (sphericalHarmonicsDegree >= 2) { - float xx = x * x; - float yy = y * y; - float zz = z * z; - float xy = x * y; - float yz = y * z; - float xz = x * z; - - vec3 sh4; - vec3 sh5; - vec3 sh6; - vec3 sh7; - vec3 sh8; - - if (sphericalHarmonicsMultiTextureMode == 0) { - vec4 sampledSH12131415 = texture(sphericalHarmonicsTexture, getDataUV(6, 3, sphericalHarmonicsTextureSize)); - vec4 sampledSH16171819 = texture(sphericalHarmonicsTexture, getDataUV(6, 4, sphericalHarmonicsTextureSize)); - vec4 sampledSH20212223 = texture(sphericalHarmonicsTexture, getDataUV(6, 5, sphericalHarmonicsTextureSize)); - sh4 = sampledSH891011.gba; - sh5 = sampledSH12131415.rgb; - sh6 = vec3(sampledSH12131415.a, sampledSH16171819.rg); - sh7 = vec3(sampledSH16171819.ba, sampledSH20212223.r); - sh8 = sampledSH20212223.gba; - } else { - vec4 sampledSH4567R = texture(sphericalHarmonicsTextureR, getDataUV(2, 1, sphericalHarmonicsTextureSize)); - vec4 sampledSH4567G = texture(sphericalHarmonicsTextureG, getDataUV(2, 1, sphericalHarmonicsTextureSize)); - vec4 sampledSH4567B = texture(sphericalHarmonicsTextureB, getDataUV(2, 1, sphericalHarmonicsTextureSize)); - sh4 = vec3(sampledSH0123R.a, sampledSH4567R.rg); - sh5 = vec3(sampledSH4567R.ba, sampledSH0123G.a); - sh6 = vec3(sampledSH4567G.rgb); - sh7 = vec3(sampledSH4567G.a, sampledSH0123B.a, sampledSH4567B.r); - sh8 = vec3(sampledSH4567B.gba); - } - - if (sphericalHarmonics8BitMode == 1) { - sh4 = sh4 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh5 = sh5 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh6 = sh6 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh7 = sh7 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh8 = sh8 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - } - - vColor.rgb += - (SH_C2[0] * xy) * sh4 + - (SH_C2[1] * yz) * sh5 + - (SH_C2[2] * (2.0 * zz - xx - yy)) * sh6 + - (SH_C2[3] * xz) * sh7 + - (SH_C2[4] * (xx - yy)) * sh8; + sh4 = sh4 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh5 = sh5 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh6 = sh6 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh7 = sh7 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh8 = sh8 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; } - `; - } - - vertexShaderSource += ` - - vColor.rgb = clamp(vColor.rgb, vec3(0.), vec3(1.)); - - } + vColor.rgb += + (SH_C2[0] * xy) * sh4 + + (SH_C2[1] * yz) * sh5 + + (SH_C2[2] * (2.0 * zz - xx - yy)) * sh6 + + (SH_C2[3] * xz) * sh7 + + (SH_C2[4] * (xx - yy)) * sh8; + } `; } vertexShaderSource += ` + + vColor.rgb = clamp(vColor.rgb, vec3(0.), vec3(1.)); - vec4 sampledCovarianceA = texture(covariancesTexture, - getDataUVF(nearestEvenIndex, 1.5, oddOffset, covariancesTextureSize)); - vec4 sampledCovarianceB = texture(covariancesTexture, - getDataUVF(nearestEvenIndex, 1.5, oddOffset + uint(1), covariancesTextureSize)); - - vec3 cov3D_M11_M12_M13 = vec3(sampledCovarianceA.rgb) * (1.0 - fOddOffset) + - vec3(sampledCovarianceA.ba, sampledCovarianceB.r) * fOddOffset; - vec3 cov3D_M22_M23_M33 = vec3(sampledCovarianceA.a, sampledCovarianceB.rg) * (1.0 - fOddOffset) + - vec3(sampledCovarianceB.gba) * fOddOffset; - - // Construct the 3D covariance matrix - mat3 Vrk = mat3( - cov3D_M11_M12_M13.x, cov3D_M11_M12_M13.y, cov3D_M11_M12_M13.z, - cov3D_M11_M12_M13.y, cov3D_M22_M23_M33.x, cov3D_M22_M23_M33.y, - cov3D_M11_M12_M13.z, cov3D_M22_M23_M33.y, cov3D_M22_M23_M33.z - ); - - mat3 J; - if (orthographicMode == 1) { - // Since the projection is linear, we don't need an approximation - J = transpose(mat3(orthoZoom, 0.0, 0.0, - 0.0, orthoZoom, 0.0, - 0.0, 0.0, 0.0)); - } else { - // Construct the Jacobian of the affine approximation of the projection matrix. It will be used to transform the - // 3D covariance matrix instead of using the actual projection matrix because that transformation would - // require a non-linear component (perspective division) which would yield a non-gaussian result. - float s = 1.0 / (viewCenter.z * viewCenter.z); - J = mat3( - focal.x / viewCenter.z, 0., -(focal.x * viewCenter.x) * s, - 0., focal.y / viewCenter.z, -(focal.y * viewCenter.y) * s, - 0., 0., 0. - ); - } - - // Concatenate the projection approximation with the model-view transformation - mat3 W = transpose(mat3(transformModelViewMatrix)); - mat3 T = W * J; - - // Transform the 3D covariance matrix (Vrk) to compute the 2D covariance matrix - mat3 cov2Dm = transpose(T) * Vrk * T; - `; - - if (antialiased) { - vertexShaderSource += ` - float detOrig = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1]; - cov2Dm[0][0] += 0.3; - cov2Dm[1][1] += 0.3; - float detBlur = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1]; - vColor.a *= sqrt(max(detOrig / detBlur, 0.0)); - if (vColor.a < minAlpha) return; - `; - } else { - vertexShaderSource += ` - cov2Dm[0][0] += 0.3; - cov2Dm[1][1] += 0.3; - `; } - vertexShaderSource += ` - - // We are interested in the upper-left 2x2 portion of the projected 3D covariance matrix because - // we only care about the X and Y values. We want the X-diagonal, cov2Dm[0][0], - // the Y-diagonal, cov2Dm[1][1], and the correlation between the two cov2Dm[0][1]. We don't - // need cov2Dm[1][0] because it is a symetric matrix. - vec3 cov2Dv = vec3(cov2Dm[0][0], cov2Dm[0][1], cov2Dm[1][1]); - - vec3 ndcCenter = clipCenter.xyz / clipCenter.w; - - // We now need to solve for the eigen-values and eigen vectors of the 2D covariance matrix - // so that we can determine the 2D basis for the splat. This is done using the method described - // here: https://people.math.harvard.edu/~knill/teaching/math21b2004/exhibits/2dmatrices/index.html - // After calculating the eigen-values and eigen-vectors, we calculate the basis for rendering the splat - // by normalizing the eigen-vectors and then multiplying them by (sqrt(8) * sqrt(eigen-value)), which is - // equal to scaling them by sqrt(8) standard deviations. - // - // This is a different approach than in the original work at INRIA. In that work they compute the - // max extents of the projected splat in screen space to form a screen-space aligned bounding rectangle - // which forms the geometry that is actually rasterized. The dimensions of that bounding box are 3.0 - // times the square root of the maximum eigen-value, or 3 standard deviations. They then use the inverse - // 2D covariance matrix (called 'conic') in the CUDA rendering thread to determine fragment opacity by - // calculating the full gaussian: exp(-0.5 * (X - mean) * conic * (X - mean)) * splat opacity - float a = cov2Dv.x; - float d = cov2Dv.z; - float b = cov2Dv.y; - float D = a * d - b * b; - float trace = a + d; - float traceOver2 = 0.5 * trace; - float term2 = sqrt(max(0.1f, traceOver2 * traceOver2 - D)); - float eigenValue1 = traceOver2 + term2; - float eigenValue2 = traceOver2 - term2; - - if (pointCloudModeEnabled == 1) { - eigenValue1 = eigenValue2 = 0.2; - } - - if (eigenValue2 <= 0.0) return; - - vec2 eigenVector1 = normalize(vec2(b, eigenValue1 - a)); - // since the eigen vectors are orthogonal, we derive the second one from the first - vec2 eigenVector2 = vec2(eigenVector1.y, -eigenVector1.x); - - // We use sqrt(8) standard deviations instead of 3 to eliminate more of the splat with a very low opacity. - vec2 basisVector1 = eigenVector1 * splatScale * min(sqrt8 * sqrt(eigenValue1), ${parseInt(maxScreenSpaceSplatSize)}.0); - vec2 basisVector2 = eigenVector2 * splatScale * min(sqrt8 * sqrt(eigenValue2), ${parseInt(maxScreenSpaceSplatSize)}.0); - - if (fadeInComplete == 0) { - float opacityAdjust = 1.0; - float centerDist = length(splatCenter - sceneCenter); - float renderTime = max(currentTime - firstRenderTime, 0.0); + `; + } - float fadeDistance = 0.75; - float distanceLoadFadeInFactor = step(visibleRegionFadeStartRadius, centerDist); - distanceLoadFadeInFactor = (1.0 - distanceLoadFadeInFactor) + - (1.0 - clamp((centerDist - visibleRegionFadeStartRadius) / fadeDistance, 0.0, 1.0)) * - distanceLoadFadeInFactor; - opacityAdjust *= distanceLoadFadeInFactor; - vColor.a *= opacityAdjust; - } - `; + return vertexShaderSource; + } - if (enableOptionalEffects) { - vertexShaderSource += ` - vColor.a *= splatOpacityFromScene; - `; + static getVertexShaderFadeIn() { + return ` + if (fadeInComplete == 0) { + float opacityAdjust = 1.0; + float centerDist = length(splatCenter - sceneCenter); + float renderTime = max(currentTime - firstRenderTime, 0.0); + + float fadeDistance = 0.75; + float distanceLoadFadeInFactor = step(visibleRegionFadeStartRadius, centerDist); + distanceLoadFadeInFactor = (1.0 - distanceLoadFadeInFactor) + + (1.0 - clamp((centerDist - visibleRegionFadeStartRadius) / fadeDistance, 0.0, 1.0)) * + distanceLoadFadeInFactor; + opacityAdjust *= distanceLoadFadeInFactor; + vColor.a *= opacityAdjust; } + `; + } - vertexShaderSource += ` - vec2 ndcOffset = vec2(vPosition.x * basisVector1 + vPosition.y * basisVector2) * - basisViewport * 2.0 * inverseFocalAdjustment; - - vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0); - gl_Position = quadPos; - - // Scale the position data we send to the fragment shader - vPosition *= sqrt8; - }`; - - const fragmentShaderSource = ` - precision highp float; - #include - - uniform vec3 debugColor; - - varying vec4 vColor; - varying vec2 vUv; - - varying vec2 vPosition; - - void main () { - // Compute the positional squared distance from the center of the splat to the current fragment. - float A = dot(vPosition, vPosition); - // Since the positional data in vPosition has been scaled by sqrt(8), the squared result will be - // scaled by a factor of 8. If the squared result is larger than 8, it means it is outside the ellipse - // defined by the rectangle formed by vPosition. It also means it's farther - // away than sqrt(8) standard deviations from the mean. - if (A > 8.0) discard; - vec3 color = vColor.rgb; - - // Since the rendered splat is scaled by sqrt(8), the inverse covariance matrix that is part of - // the gaussian formula becomes the identity matrix. We're then left with (X - mean) * (X - mean), - // and since 'mean' is zero, we have X * X, which is the same as A: - float opacity = exp(-0.5 * A) * vColor.a; - - gl_FragColor = vec4(color.rgb, opacity); - }`; + static getUniforms(dynamicMode = false, enableOptionalEffects = false, maxSphericalHarmonicsDegree = 0, + splatScale = 1.0, pointCloudModeEnabled = false) { const uniforms = { 'sceneCenter': { @@ -518,10 +372,6 @@ export class SplatMaterial { 'type': 'f', 'value': 0.0 }, - 'covariancesTexture': { - 'type': 't', - 'value': null - }, 'centersColorsTexture': { 'type': 't', 'value': null @@ -566,10 +416,6 @@ export class SplatMaterial { 'type': 'v3', 'value': new THREE.Color() }, - 'covariancesTextureSize': { - 'type': 'v2', - 'value': new THREE.Vector2(1024, 1024) - }, 'centersColorsTextureSize': { 'type': 'v2', 'value': new THREE.Vector2(1024, 1024) @@ -642,19 +488,7 @@ export class SplatMaterial { }; } - const material = new THREE.ShaderMaterial({ - uniforms: uniforms, - vertexShader: vertexShaderSource, - fragmentShader: fragmentShaderSource, - transparent: true, - alphaTest: 1.0, - blending: THREE.NormalBlending, - depthTest: true, - depthWrite: false, - side: THREE.DoubleSide - }); - - return material; + return uniforms; } } diff --git a/src/splatmesh/SplatMaterial2D.js b/src/splatmesh/SplatMaterial2D.js new file mode 100644 index 00000000..941c22c6 --- /dev/null +++ b/src/splatmesh/SplatMaterial2D.js @@ -0,0 +1,344 @@ +import * as THREE from 'three'; +import { SplatMaterial } from './SplatMaterial.js'; + +export class SplatMaterial2D { + + /** + * Build the Three.js material that is used to render the splats. + * @param {number} dynamicMode If true, it means the scene geometry represented by this splat mesh is not stationary or + * that the splat count might change + * @param {boolean} enableOptionalEffects When true, allows for usage of extra properties and attributes in the shader for effects + * such as opacity adjustment. Default is false for performance reasons. + * @param {number} splatScale Value by which all splats are scaled in screen-space (default is 1.0) + * @param {number} pointCloudModeEnabled Render all splats as screen-space circles + * @param {number} maxSphericalHarmonicsDegree Degree of spherical harmonics to utilize in rendering splats + * @return {THREE.ShaderMaterial} + */ + static build(dynamicMode = false, enableOptionalEffects = false, splatScale = 1.0, + pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0) { + + const customVertexVars = ` + uniform vec2 scaleRotationsTextureSize; + uniform highp sampler2D scaleRotationsTexture; + varying mat3 vT; + varying vec2 vQuadCenter; + varying vec2 vFragCoord; + `; + + let vertexShaderSource = SplatMaterial.buildVertexShaderBase(dynamicMode, enableOptionalEffects, + maxSphericalHarmonicsDegree, customVertexVars); + vertexShaderSource += SplatMaterial2D.buildVertexShaderProjection(); + const fragmentShaderSource = SplatMaterial2D.buildFragmentShader(); + + const uniforms = SplatMaterial.getUniforms(dynamicMode, enableOptionalEffects, + maxSphericalHarmonicsDegree, splatScale, pointCloudModeEnabled); + + uniforms['scaleRotationsTexture'] = { + 'type': 't', + 'value': null + }; + uniforms['scaleRotationsTextureSize'] = { + 'type': 'v2', + 'value': new THREE.Vector2(1024, 1024) + }; + + const material = new THREE.ShaderMaterial({ + uniforms: uniforms, + vertexShader: vertexShaderSource, + fragmentShader: fragmentShaderSource, + transparent: true, + alphaTest: 1.0, + blending: THREE.NormalBlending, + depthTest: true, + depthWrite: false, + side: THREE.DoubleSide + }); + + return material; + } + + static buildVertexShaderProjection() { + /* + glm::mat3 R = quat_to_rotmat(rot); + glm::mat3 S = scale_to_mat(scale, mod); + glm::mat3 L = R * S; + + // center of Gaussians in the camera coordinate + glm::mat3x4 splat2world = glm::mat3x4( + glm::vec4(L[0], 0.0), + glm::vec4(L[1], 0.0), + glm::vec4(p_orig.x, p_orig.y, p_orig.z, 1) + ); + + glm::mat4 world2ndc = glm::mat4( + projmatrix[0], projmatrix[4], projmatrix[8], projmatrix[12], + projmatrix[1], projmatrix[5], projmatrix[9], projmatrix[13], + projmatrix[2], projmatrix[6], projmatrix[10], projmatrix[14], + projmatrix[3], projmatrix[7], projmatrix[11], projmatrix[15] + ); + + glm::mat3x4 ndc2pix = glm::mat3x4( + glm::vec4(float(W) / 2.0, 0.0, 0.0, float(W-1) / 2.0), + glm::vec4(0.0, float(H) / 2.0, 0.0, float(H-1) / 2.0), + glm::vec4(0.0, 0.0, 0.0, 1.0) + ); + + T = glm::transpose(splat2world) * world2ndc * ndc2pix; + normal = transformVec4x3({L[2].x, L[2].y, L[2].z}, viewmatrix); + + */ + + // Compute a 2D-to-2D mapping matrix from a tangent plane into a image plane + // given a 2D gaussian parameters. T = WH (from the paper: https://arxiv.org/pdf/2403.17888) + let vertexShaderSource = ` + + vec4 scaleRotationA = texture(scaleRotationsTexture, getDataUVF(nearestEvenIndex, 1.5, + oddOffset, scaleRotationsTextureSize)); + vec4 scaleRotationB = texture(scaleRotationsTexture, getDataUVF(nearestEvenIndex, 1.5, + oddOffset + uint(1), scaleRotationsTextureSize)); + + vec3 scaleRotation123 = vec3(scaleRotationA.rgb) * (1.0 - fOddOffset) + + vec3(scaleRotationA.ba, scaleRotationB.r) * fOddOffset; + vec3 scaleRotation456 = vec3(scaleRotationA.a, scaleRotationB.rg) * (1.0 - fOddOffset) + + vec3(scaleRotationB.gba) * fOddOffset; + + float missingW = sqrt(1.0 - scaleRotation456.x * scaleRotation456.x - scaleRotation456.y * + scaleRotation456.y - scaleRotation456.z * scaleRotation456.z); + mat3 R = quaternionToRotationMatrix(scaleRotation456.r, scaleRotation456.g, scaleRotation456.b, missingW); + mat3 S = mat3(scaleRotation123.r, 0.0, 0.0, + 0.0, scaleRotation123.g, 0.0, + 0.0, 0.0, scaleRotation123.b); + + mat3 L = R * S; + + mat3x4 splat2World = mat3x4(vec4(L[0], 0.0), + vec4(L[1], 0.0), + vec4(splatCenter.x, splatCenter.y, splatCenter.z, 1.0)); + + mat4 world2ndc = transpose(projectionMatrix * transformModelViewMatrix); + + mat3x4 ndc2pix = mat3x4(vec4(viewport.x / 2.0, 0.0, 0.0, (viewport.x - 1.0) / 2.0), + vec4(0.0, viewport.y / 2.0, 0.0, (viewport.y - 1.0) / 2.0), + vec4(0.0, 0.0, 0.0, 1.0)); + + mat3 T = transpose(splat2World) * world2ndc * ndc2pix; + vec3 normal = vec3(viewMatrix * vec4(L[0][2], L[1][2], L[2][2], 0.0)); + `; + + /* + float3 T0 = {T[0][0], T[0][1], T[0][2]}; + float3 T1 = {T[1][0], T[1][1], T[1][2]}; + float3 T3 = {T[2][0], T[2][1], T[2][2]}; + + // Compute AABB + float3 temp_point = {1.0f, 1.0f, -1.0f}; + float distance = sumf3(T3 * T3 * temp_point); + float3 f = (1 / distance) * temp_point; + if (distance == 0.0) return false; + + point_image = { + sumf3(f * T0 * T3), + sumf3(f * T1 * T3) + }; + + float2 temp = { + sumf3(f * T0 * T0), + sumf3(f * T1 * T1) + }; + float2 half_extend = point_image * point_image - temp; + extent = sqrtf2(maxf2(1e-4, half_extend)); + return true; + */ + // Computing the bounding box of the 2D Gaussian and its center + // The center of the bounding box is used to create a low pass filter. + // This code is based off the reference implementation and creates an AABB aligned + // with the screen for the quad to be rendered. + const referenceQuadGeneration = ` + vec3 T0 = vec3(T[0][0], T[0][1], T[0][2]); + vec3 T1 = vec3(T[1][0], T[1][1], T[1][2]); + vec3 T3 = vec3(T[2][0], T[2][1], T[2][2]); + + vec3 tempPoint = vec3(1.0, 1.0, -1.0); + float distance = (T3.x * T3.x * tempPoint.x) + (T3.y * T3.y * tempPoint.y) + (T3.z * T3.z * tempPoint.z); + vec3 f = (1.0 / distance) * tempPoint; + if (abs(distance) < 0.00001) return; + + float pointImageX = (T0.x * T3.x * f.x) + (T0.y * T3.y * f.y) + (T0.z * T3.z * f.z); + float pointImageY = (T1.x * T3.x * f.x) + (T1.y * T3.y * f.y) + (T1.z * T3.z * f.z); + vec2 pointImage = vec2(pointImageX, pointImageY); + + float tempX = (T0.x * T0.x * f.x) + (T0.y * T0.y * f.y) + (T0.z * T0.z * f.z); + float tempY = (T1.x * T1.x * f.x) + (T1.y * T1.y * f.y) + (T1.z * T1.z * f.z); + vec2 temp = vec2(tempX, tempY); + + vec2 halfExtend = pointImage * pointImage - temp; + vec2 extent = sqrt(max(vec2(0.0001), halfExtend)); + float radius = max(extent.x, extent.y); + + vec2 ndcOffset = ((position.xy * radius * 3.0) * basisViewport * 2.0); + + vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0); + gl_Position = quadPos; + + vT = T; + vQuadCenter = pointImage; + vFragCoord = (quadPos.xy * 0.5 + 0.5) * viewport; + `; + + const useRefImplementation = false; + if (useRefImplementation) { + vertexShaderSource += referenceQuadGeneration; + } else { + // Create a quad that is aligned with the eigen vectors of the projected gaussian for rendering. + // This is a different approach than the reference implementation, similar to how the rendering of + // 3D gaussians in this viewer differs from the reference implementation. If the quad is too small + // (smaller than a pixel), then revert to the reference implementation. + vertexShaderSource += ` + + mat4 splat2World4 = mat4(vec4(L[0], 0.0), + vec4(L[1], 0.0), + vec4(L[2], 0.0), + vec4(splatCenter.x, splatCenter.y, splatCenter.z, 1.0)); + + mat4 Tt = transpose(transpose(splat2World4) * world2ndc); + + vec4 tempPoint1 = Tt * vec4(1.0, 0.0, 0.0, 1.0); + tempPoint1 /= tempPoint1.w; + + vec4 tempPoint2 = Tt * vec4(0.0, 1.0, 0.0, 1.0); + tempPoint2 /= tempPoint2.w; + + vec4 center = Tt * vec4(0.0, 0.0, 0.0, 1.0); + center /= center.w; + + vec2 basisVector1 = tempPoint1.xy - center.xy; + vec2 basisVector2 = tempPoint2.xy - center.xy; + + vec2 basisVector1Screen = basisVector1 * 0.5 * viewport; + vec2 basisVector2Screen = basisVector2 * 0.5 * viewport; + + const float minPix = 1.; + if (length(basisVector1Screen) < minPix || length(basisVector2Screen) < minPix) { + ${referenceQuadGeneration} + } else { + vec2 ndcOffset = vec2(position.x * basisVector1 + position.y * basisVector2) * 3.0 * inverseFocalAdjustment; + vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0); + gl_Position = quadPos; + + vT = T; + vQuadCenter = center.xy; + vFragCoord = (quadPos.xy * 0.5 + 0.5) * viewport; + } + `; + } + + vertexShaderSource += SplatMaterial.getVertexShaderFadeIn(); + vertexShaderSource += `}`; + + return vertexShaderSource; + } + + static buildFragmentShader() { + + let fragmentShaderSource = ` + precision highp float; + #include + + uniform vec3 debugColor; + + varying vec4 vColor; + varying vec2 vUv; + varying vec2 vPosition; + varying mat3 vT; + varying vec2 vQuadCenter; + varying vec2 vFragCoord; + + /* + const float2 xy = collected_xy[j]; + const float3 Tu = collected_Tu[j]; + const float3 Tv = collected_Tv[j]; + const float3 Tw = collected_Tw[j]; + float3 k = pix.x * Tw - Tu; + float3 l = pix.y * Tw - Tv; + float3 p = cross(k, l); + if (p.z == 0.0) continue; + float2 s = {p.x / p.z, p.y / p.z}; + float rho3d = (s.x * s.x + s.y * s.y); + float2 d = {xy.x - pixf.x, xy.y - pixf.y}; + float rho2d = FilterInvSquare * (d.x * d.x + d.y * d.y); + + // compute intersection and depth + float rho = min(rho3d, rho2d); + float depth = (rho3d <= rho2d) ? (s.x * Tw.x + s.y * Tw.y) + Tw.z : Tw.z; + if (depth < near_n) continue; + float4 nor_o = collected_normal_opacity[j]; + float normal[3] = {nor_o.x, nor_o.y, nor_o.z}; + float opa = nor_o.w; + + float power = -0.5f * rho; + if (power > 0.0f) + continue; + + // Eq. (2) from 3D Gaussian splatting paper. + // Obtain alpha by multiplying with Gaussian opacity + // and its exponential falloff from mean. + // Avoid numerical instabilities (see paper appendix). + float alpha = min(0.99f, opa * exp(power)); + if (alpha < 1.0f / 255.0f) + continue; + float test_T = T * (1 - alpha); + if (test_T < 0.0001f) + { + done = true; + continue; + } + + float w = alpha * T; + */ + void main () { + + const float FilterInvSquare = 2.0; + const float near_n = 0.2; + const float T = 1.0; + + vec2 xy = vQuadCenter; + vec3 Tu = vT[0]; + vec3 Tv = vT[1]; + vec3 Tw = vT[2]; + vec3 k = vFragCoord.x * Tw - Tu; + vec3 l = vFragCoord.y * Tw - Tv; + vec3 p = cross(k, l); + if (p.z == 0.0) discard; + vec2 s = vec2(p.x / p.z, p.y / p.z); + float rho3d = (s.x * s.x + s.y * s.y); + vec2 d = vec2(xy.x - vFragCoord.x, xy.y - vFragCoord.y); + float rho2d = FilterInvSquare * (d.x * d.x + d.y * d.y); + + // compute intersection and depth + float rho = min(rho3d, rho2d); + float depth = (rho3d <= rho2d) ? (s.x * Tw.x + s.y * Tw.y) + Tw.z : Tw.z; + if (depth < near_n) discard; + // vec4 nor_o = collected_normal_opacity[j]; + // float normal[3] = {nor_o.x, nor_o.y, nor_o.z}; + float opa = vColor.a; + + float power = -0.5f * rho; + if (power > 0.0f) discard; + + // Eq. (2) from 3D Gaussian splatting paper. + // Obtain alpha by multiplying with Gaussian opacity + // and its exponential falloff from mean. + // Avoid numerical instabilities (see paper appendix). + float alpha = min(0.99f, opa * exp(power)); + if (alpha < 1.0f / 255.0f) discard; + float test_T = T * (1.0 - alpha); + if (test_T < 0.0001)discard; + + float w = alpha * T; + gl_FragColor = vec4(vColor.rgb, w); + } + `; + + return fragmentShaderSource; + } +} diff --git a/src/splatmesh/SplatMaterial3D.js b/src/splatmesh/SplatMaterial3D.js new file mode 100644 index 00000000..7249fa00 --- /dev/null +++ b/src/splatmesh/SplatMaterial3D.js @@ -0,0 +1,256 @@ +import * as THREE from 'three'; +import { SplatMaterial } from './SplatMaterial.js'; + +export class SplatMaterial3D { + + /** + * Build the Three.js material that is used to render the splats. + * @param {number} dynamicMode If true, it means the scene geometry represented by this splat mesh is not stationary or + * that the splat count might change + * @param {boolean} enableOptionalEffects When true, allows for usage of extra properties and attributes in the shader for effects + * such as opacity adjustment. Default is false for performance reasons. + * @param {boolean} antialiased If true, calculate compensation factor to deal with gaussians being rendered at a significantly + * different resolution than that of their training + * @param {number} maxScreenSpaceSplatSize The maximum clip space splat size + * @param {number} splatScale Value by which all splats are scaled in screen-space (default is 1.0) + * @param {number} pointCloudModeEnabled Render all splats as screen-space circles + * @param {number} maxSphericalHarmonicsDegree Degree of spherical harmonics to utilize in rendering splats + * @return {THREE.ShaderMaterial} + */ + static build(dynamicMode = false, enableOptionalEffects = false, antialiased = false, + maxScreenSpaceSplatSize = 2048, splatScale = 1.0, pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0) { + + const customVertexVars = ` + uniform vec2 covariancesTextureSize; + uniform highp sampler2D covariancesTexture; + uniform highp usampler2D covariancesTextureHalfFloat; + uniform int covariancesAreHalfFloat; + + void fromCovarianceHalfFloatV4(uvec4 val, out vec4 first, out vec4 second) { + vec2 r = unpackHalf2x16(val.r); + vec2 g = unpackHalf2x16(val.g); + vec2 b = unpackHalf2x16(val.b); + + first = vec4(r.x, r.y, g.x, g.y); + second = vec4(b.x, b.y, 0.0, 0.0); + } + `; + + let vertexShaderSource = SplatMaterial.buildVertexShaderBase(dynamicMode, enableOptionalEffects, + maxSphericalHarmonicsDegree, customVertexVars); + vertexShaderSource += SplatMaterial3D.buildVertexShaderProjection(antialiased, enableOptionalEffects, maxScreenSpaceSplatSize); + const fragmentShaderSource = SplatMaterial3D.buildFragmentShader(); + + const uniforms = SplatMaterial.getUniforms(dynamicMode, enableOptionalEffects, + maxSphericalHarmonicsDegree, splatScale, pointCloudModeEnabled); + + uniforms['covariancesTextureSize'] = { + 'type': 'v2', + 'value': new THREE.Vector2(1024, 1024) + }; + uniforms['covariancesTexture'] = { + 'type': 't', + 'value': null + }; + uniforms['covariancesTextureHalfFloat'] = { + 'type': 't', + 'value': null + }; + uniforms['covariancesAreHalfFloat'] = { + 'type': 'i', + 'value': 0 + }; + + const material = new THREE.ShaderMaterial({ + uniforms: uniforms, + vertexShader: vertexShaderSource, + fragmentShader: fragmentShaderSource, + transparent: true, + alphaTest: 1.0, + blending: THREE.NormalBlending, + depthTest: true, + depthWrite: false, + side: THREE.DoubleSide + }); + + return material; + } + + static buildVertexShaderProjection(antialiased, enableOptionalEffects, maxScreenSpaceSplatSize) { + let vertexShaderSource = ` + + vec4 sampledCovarianceA; + vec4 sampledCovarianceB; + vec3 cov3D_M11_M12_M13; + vec3 cov3D_M22_M23_M33; + if (covariancesAreHalfFloat == 0) { + sampledCovarianceA = texture(covariancesTexture, getDataUVF(nearestEvenIndex, 1.5, oddOffset, + covariancesTextureSize)); + sampledCovarianceB = texture(covariancesTexture, getDataUVF(nearestEvenIndex, 1.5, oddOffset + uint(1), + covariancesTextureSize)); + + cov3D_M11_M12_M13 = vec3(sampledCovarianceA.rgb) * (1.0 - fOddOffset) + + vec3(sampledCovarianceA.ba, sampledCovarianceB.r) * fOddOffset; + cov3D_M22_M23_M33 = vec3(sampledCovarianceA.a, sampledCovarianceB.rg) * (1.0 - fOddOffset) + + vec3(sampledCovarianceB.gba) * fOddOffset; + } else { + uvec4 sampledCovarianceU = texture(covariancesTextureHalfFloat, getDataUV(1, 0, covariancesTextureSize)); + fromCovarianceHalfFloatV4(sampledCovarianceU, sampledCovarianceA, sampledCovarianceB); + cov3D_M11_M12_M13 = sampledCovarianceA.rgb; + cov3D_M22_M23_M33 = vec3(sampledCovarianceA.a, sampledCovarianceB.rg); + } + + // Construct the 3D covariance matrix + mat3 Vrk = mat3( + cov3D_M11_M12_M13.x, cov3D_M11_M12_M13.y, cov3D_M11_M12_M13.z, + cov3D_M11_M12_M13.y, cov3D_M22_M23_M33.x, cov3D_M22_M23_M33.y, + cov3D_M11_M12_M13.z, cov3D_M22_M23_M33.y, cov3D_M22_M23_M33.z + ); + + mat3 J; + if (orthographicMode == 1) { + // Since the projection is linear, we don't need an approximation + J = transpose(mat3(orthoZoom, 0.0, 0.0, + 0.0, orthoZoom, 0.0, + 0.0, 0.0, 0.0)); + } else { + // Construct the Jacobian of the affine approximation of the projection matrix. It will be used to transform the + // 3D covariance matrix instead of using the actual projection matrix because that transformation would + // require a non-linear component (perspective division) which would yield a non-gaussian result. + float s = 1.0 / (viewCenter.z * viewCenter.z); + J = mat3( + focal.x / viewCenter.z, 0., -(focal.x * viewCenter.x) * s, + 0., focal.y / viewCenter.z, -(focal.y * viewCenter.y) * s, + 0., 0., 0. + ); + } + + // Concatenate the projection approximation with the model-view transformation + mat3 W = transpose(mat3(transformModelViewMatrix)); + mat3 T = W * J; + + // Transform the 3D covariance matrix (Vrk) to compute the 2D covariance matrix + mat3 cov2Dm = transpose(T) * Vrk * T; + `; + + if (antialiased) { + vertexShaderSource += ` + float detOrig = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1]; + cov2Dm[0][0] += 0.3; + cov2Dm[1][1] += 0.3; + float detBlur = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1]; + vColor.a *= sqrt(max(detOrig / detBlur, 0.0)); + if (vColor.a < minAlpha) return; + `; + } else { + vertexShaderSource += ` + cov2Dm[0][0] += 0.3; + cov2Dm[1][1] += 0.3; + `; + } + + vertexShaderSource += ` + + // We are interested in the upper-left 2x2 portion of the projected 3D covariance matrix because + // we only care about the X and Y values. We want the X-diagonal, cov2Dm[0][0], + // the Y-diagonal, cov2Dm[1][1], and the correlation between the two cov2Dm[0][1]. We don't + // need cov2Dm[1][0] because it is a symetric matrix. + vec3 cov2Dv = vec3(cov2Dm[0][0], cov2Dm[0][1], cov2Dm[1][1]); + + // We now need to solve for the eigen-values and eigen vectors of the 2D covariance matrix + // so that we can determine the 2D basis for the splat. This is done using the method described + // here: https://people.math.harvard.edu/~knill/teaching/math21b2004/exhibits/2dmatrices/index.html + // After calculating the eigen-values and eigen-vectors, we calculate the basis for rendering the splat + // by normalizing the eigen-vectors and then multiplying them by (sqrt(8) * sqrt(eigen-value)), which is + // equal to scaling them by sqrt(8) standard deviations. + // + // This is a different approach than in the original work at INRIA. In that work they compute the + // max extents of the projected splat in screen space to form a screen-space aligned bounding rectangle + // which forms the geometry that is actually rasterized. The dimensions of that bounding box are 3.0 + // times the square root of the maximum eigen-value, or 3 standard deviations. They then use the inverse + // 2D covariance matrix (called 'conic') in the CUDA rendering thread to determine fragment opacity by + // calculating the full gaussian: exp(-0.5 * (X - mean) * conic * (X - mean)) * splat opacity + float a = cov2Dv.x; + float d = cov2Dv.z; + float b = cov2Dv.y; + float D = a * d - b * b; + float trace = a + d; + float traceOver2 = 0.5 * trace; + float term2 = sqrt(max(0.1f, traceOver2 * traceOver2 - D)); + float eigenValue1 = traceOver2 + term2; + float eigenValue2 = traceOver2 - term2; + + if (pointCloudModeEnabled == 1) { + eigenValue1 = eigenValue2 = 0.2; + } + + if (eigenValue2 <= 0.0) return; + + vec2 eigenVector1 = normalize(vec2(b, eigenValue1 - a)); + // since the eigen vectors are orthogonal, we derive the second one from the first + vec2 eigenVector2 = vec2(eigenVector1.y, -eigenVector1.x); + + // We use sqrt(8) standard deviations instead of 3 to eliminate more of the splat with a very low opacity. + vec2 basisVector1 = eigenVector1 * splatScale * min(sqrt8 * sqrt(eigenValue1), ${parseInt(maxScreenSpaceSplatSize)}.0); + vec2 basisVector2 = eigenVector2 * splatScale * min(sqrt8 * sqrt(eigenValue2), ${parseInt(maxScreenSpaceSplatSize)}.0); + `; + + if (enableOptionalEffects) { + vertexShaderSource += ` + vColor.a *= splatOpacityFromScene; + `; + } + + vertexShaderSource += ` + vec2 ndcOffset = vec2(vPosition.x * basisVector1 + vPosition.y * basisVector2) * + basisViewport * 2.0 * inverseFocalAdjustment; + + vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0); + gl_Position = quadPos; + + // Scale the position data we send to the fragment shader + vPosition *= sqrt8; + `; + + vertexShaderSource += SplatMaterial.getVertexShaderFadeIn(); + vertexShaderSource += `}`; + + return vertexShaderSource; + } + + static buildFragmentShader() { + let fragmentShaderSource = ` + precision highp float; + #include + + uniform vec3 debugColor; + + varying vec4 vColor; + varying vec2 vUv; + varying vec2 vPosition; + `; + + fragmentShaderSource += ` + void main () { + // Compute the positional squared distance from the center of the splat to the current fragment. + float A = dot(vPosition, vPosition); + // Since the positional data in vPosition has been scaled by sqrt(8), the squared result will be + // scaled by a factor of 8. If the squared result is larger than 8, it means it is outside the ellipse + // defined by the rectangle formed by vPosition. It also means it's farther + // away than sqrt(8) standard deviations from the mean. + if (A > 8.0) discard; + vec3 color = vColor.rgb; + + // Since the rendered splat is scaled by sqrt(8), the inverse covariance matrix that is part of + // the gaussian formula becomes the identity matrix. We're then left with (X - mean) * (X - mean), + // and since 'mean' is zero, we have X * X, which is the same as A: + float opacity = exp(-0.5 * A) * vColor.a; + + gl_FragColor = vec4(color.rgb, opacity); + } + `; + + return fragmentShaderSource; + } + +} diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index ac849ddd..1ab21805 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -1,5 +1,6 @@ import * as THREE from 'three'; -import { SplatMaterial } from './SplatMaterial.js'; +import { SplatMaterial3D } from './SplatMaterial3D.js'; +import { SplatMaterial2D } from './SplatMaterial2D.js'; import { SplatGeometry } from './SplatGeometry.js'; import { SplatScene } from './SplatScene.js'; import { SplatTree } from '../splattree/SplatTree.js'; @@ -8,6 +9,7 @@ import { WebGLCapabilities } from '../three-shim/WebGLCapabilities.js'; import { uintEncodedFloat, rgbaArrayToInteger } from '../Util.js'; import { Constants } from '../Constants.js'; import { SceneRevealMode } from '../SceneRevealMode.js'; +import { SplatRenderMode } from '../SplatRenderMode.js'; import { LogLevel } from '../LogLevel.js'; import { clamp, getSphericalHarmonicsComponentCountForDegree } from '../Util.js'; @@ -17,7 +19,11 @@ const dummyMaterial = new THREE.MeshBasicMaterial(); const COVARIANCES_ELEMENTS_PER_SPLAT = 6; const CENTER_COLORS_ELEMENTS_PER_SPLAT = 4; -const COVARIANCES_ELEMENTS_PER_TEXEL = 4; +const COVARIANCES_ELEMENTS_PER_TEXEL_STORED = 4; +const COVARIANCES_ELEMENTS_PER_TEXEL_ALLOCATED = 4; +const COVARIANCES_ELEMENTS_PER_TEXEL_COMPRESSED_STORED = 6; +const COVARIANCES_ELEMENTS_PER_TEXEL_COMPRESSED_ALLOCATED = 8; +const SCALES_ROTATIONS_ELEMENTS_PER_TEXEL = 4; const CENTER_COLORS_ELEMENTS_PER_TEXEL = 4; const SCENE_INDEXES_ELEMENTS_PER_TEXEL = 1; @@ -26,22 +32,31 @@ const SCENE_FADEIN_RATE_GRADUAL = 0.003; const VISIBLE_REGION_EXPANSION_DELTA = 1; +// Based on my own observations across multiple devices, OSes and browsers, using textures that have one dimension +// greater than 4096 while the other is greater than or equal to 4096 causes issues (Essentially any texture larger +// than 4096 x 4096 (16777216) texels). Specifically it seems all texture data beyond the 4096 x 4096 texel boundary +// is corrupted, while data below that boundary is usable. In these cases the texture has been valid in the eyes of +// both Three.js and WebGL, and the texel format (RG, RGBA, etc.) has not mattered. More investigation will be needed, +// but for now the work-around is to split the spherical harmonics into three textures (one for each color channel). +const MAX_TEXTURE_TEXELS = 16777216; + /** * SplatMesh: Container for one or more splat scenes, abstracting them into a single unified container for * splat data. Additionally contains data structures and code to make the splat data renderable as a Three.js mesh. */ export class SplatMesh extends THREE.Mesh { - constructor(dynamicMode = true, enableOptionalEffects = false, halfPrecisionCovariancesOnGPU = false, devicePixelRatio = 1, - enableDistancesComputationOnGPU = true, integerBasedDistancesComputation = false, - antialiased = false, maxScreenSpaceSplatSize = 1024, logLevel = LogLevel.None, sphericalHarmonicsDegree = 0) { + constructor(splatRenderMode = SplatRenderMode.ThreeD, dynamicMode = true, enableOptionalEffects = false, + halfPrecisionCovariancesOnGPU = false, devicePixelRatio = 1, enableDistancesComputationOnGPU = true, + integerBasedDistancesComputation = false, antialiased = false, maxScreenSpaceSplatSize = 1024, logLevel = LogLevel.None, + sphericalHarmonicsDegree = 0) { super(dummyGeometry, dummyMaterial); // Reference to a Three.js renderer this.renderer = undefined; - // Use 16-bit floating point values when storing splat covariance data in textures, instead of 32-bit - this.halfPrecisionCovariancesOnGPU = halfPrecisionCovariancesOnGPU; + // Determine how the splats are rendered + this.splatRenderMode = splatRenderMode; // When 'dynamicMode' is true, scenes are assumed to be non-static. Dynamic scenes are handled differently // and certain optimizations cannot be made for them. Additionally, by default, all splat data retrieved from @@ -49,6 +64,14 @@ export class SplatMesh extends THREE.Mesh { // can be overriden via parameters to the individual functions that are used to retrieve splat data. this.dynamicMode = dynamicMode; + // When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. + // Default is false for performance reasons. These properties are separate from transform properties (scale, rotation, position) + // that are enabled by the 'dynamicScene' parameter. + this.enableOptionalEffects = enableOptionalEffects; + + // Use 16-bit floating point values when storing splat covariance data in textures, instead of 32-bit + this.halfPrecisionCovariancesOnGPU = halfPrecisionCovariancesOnGPU; + // Ratio of the resolution in physical pixels to the resolution in CSS pixels for the current display device this.devicePixelRatio = devicePixelRatio; @@ -75,11 +98,6 @@ export class SplatMesh extends THREE.Mesh { this.sphericalHarmonicsDegree = sphericalHarmonicsDegree; this.minSphericalHarmonicsDegree = 0; - // When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. - // Default is false for performance reasons. These properties are separate from transform properties (scale, rotation, position) - // that are enabled by the 'dynamicScene' parameter. - this.enableOptionalEffects = enableOptionalEffects; - // The individual splat scenes stored in this splat mesh, each containing their own transform this.scenes = []; @@ -158,13 +176,14 @@ export class SplatMesh extends THREE.Mesh { const position = new THREE.Vector3().fromArray(positionArray); const rotation = new THREE.Quaternion().fromArray(rotationArray); const scale = new THREE.Vector3().fromArray(scaleArray); - scenes[i] = SplatMesh.createScene(splatBuffer, position, rotation, scale, options.splatAlphaRemovalThreshold || 1); + scenes[i] = SplatMesh.createScene(splatBuffer, position, rotation, scale, + options.splatAlphaRemovalThreshold || 1, options.opacity, options.visible); } return scenes; } - static createScene(splatBuffer, position, rotation, scale, minimumAlpha) { - return new SplatScene(splatBuffer, position, rotation, scale, minimumAlpha); + static createScene(splatBuffer, position, rotation, scale, minimumAlpha, opacity = 1.0, visible = true) { + return new SplatScene(splatBuffer, position, rotation, scale, minimumAlpha, opacity, visible); } /** @@ -338,9 +357,15 @@ export class SplatMesh extends THREE.Mesh { this.lastBuildMaxSplatCount = 0; this.disposeMeshData(); this.geometry = SplatGeometry.build(maxSplatCount); - this.material = SplatMaterial.build(this.dynamicMode, this.enableOptionalEffects, this.antialiased, - this.maxScreenSpaceSplatSize, this.splatScale, this.pointCloudModeEnabled, - this.minSphericalHarmonicsDegree); + if (this.splatRenderMode === SplatRenderMode.ThreeD) { + this.material = SplatMaterial3D.build(this.dynamicMode, this.enableOptionalEffects, this.antialiased, + this.maxScreenSpaceSplatSize, this.splatScale, this.pointCloudModeEnabled, + this.minSphericalHarmonicsDegree); + } else { + this.material = SplatMaterial2D.build(this.dynamicMode, this.enableOptionalEffects, + this.splatScale, this.pointCloudModeEnabled, this.minSphericalHarmonicsDegree); + } + const indexMaps = SplatMesh.buildSplatIndexMaps(splatBuffers); this.globalSplatIndexToLocalSplatIndexMap = indexMaps.localSplatIndexMap; this.globalSplatIndexToSceneIndexMap = indexMaps.sceneIndexMap; @@ -362,6 +387,7 @@ export class SplatMesh extends THREE.Mesh { onSplatTreeIndexesUpload, onSplatTreeConstruction) .then(() => { if (this.onSplatTreeReadyCallback) this.onSplatTreeReadyCallback(this.splatTree); + this.onSplatTreeReadyCallback = null; }); } @@ -513,7 +539,8 @@ export class SplatMesh extends THREE.Mesh { if (this.splatTree) { this.splatTree.dispose(); this.splatTree = null; - } else if (this.baseSplatTree) { + } + if (this.baseSplatTree) { this.baseSplatTree.dispose(); this.baseSplatTree = null; } @@ -608,10 +635,34 @@ export class SplatMesh extends THREE.Mesh { return texSize; }; - const covarianceCompressionLevel = this.getTargetCovarianceCompressionLevel(); + const getCovariancesElementsPertexelStored = (compressionLevel) => { + return compressionLevel >= 1 ? COVARIANCES_ELEMENTS_PER_TEXEL_COMPRESSED_STORED : COVARIANCES_ELEMENTS_PER_TEXEL_STORED; + }; + + const getCovariancesInitialTextureSpecs = (compressionLevel) => { + const elementsPerTexelStored = getCovariancesElementsPertexelStored(compressionLevel); + const texSize = computeDataTextureSize(elementsPerTexelStored, 6); + return {elementsPerTexelStored, texSize}; + }; + + let covarianceCompressionLevel = this.getTargetCovarianceCompressionLevel(); + const scaleRotationCompressionLevel = 0; const shCompressionLevel = this.getTargetSphericalHarmonicsCompressionLevel(); - const covariances = new Float32Array(maxSplatCount * COVARIANCES_ELEMENTS_PER_SPLAT); + let covariances; + let scales; + let rotations; + if (this.splatRenderMode === SplatRenderMode.ThreeD) { + const initialCovTexSpecs = getCovariancesInitialTextureSpecs(covarianceCompressionLevel); + if (initialCovTexSpecs.texSize.x * initialCovTexSpecs.texSize.y > MAX_TEXTURE_TEXELS && covarianceCompressionLevel === 0) { + covarianceCompressionLevel = 1; + } + covariances = new Float32Array(maxSplatCount * COVARIANCES_ELEMENTS_PER_SPLAT); + } else { + scales = new Float32Array(maxSplatCount * 3); + rotations = new Float32Array(maxSplatCount * 4); + } + const centers = new Float32Array(maxSplatCount * 3); const colors = new Uint8Array(maxSplatCount * 4); @@ -621,20 +672,8 @@ export class SplatMesh extends THREE.Mesh { const shComponentCount = getSphericalHarmonicsComponentCountForDegree(this.minSphericalHarmonicsDegree); const shData = this.minSphericalHarmonicsDegree ? new SphericalHarmonicsArrayType(maxSplatCount * shComponentCount) : undefined; - this.fillSplatDataArrays(covariances, centers, colors, shData, undefined, - covarianceCompressionLevel, shCompressionLevel); - - // set up covariances data texture - const covTexSize = computeDataTextureSize(COVARIANCES_ELEMENTS_PER_TEXEL, 6); - let CovariancesDataType = covarianceCompressionLevel >= 1 ? Uint16Array : Float32Array; - let covariancesTextureType = covarianceCompressionLevel >= 1 ? THREE.HalfFloatType : THREE.FloatType; - const paddedCovariances = new CovariancesDataType(covTexSize.x * covTexSize.y * COVARIANCES_ELEMENTS_PER_TEXEL); - paddedCovariances.set(covariances); - - const covTex = new THREE.DataTexture(paddedCovariances, covTexSize.x, covTexSize.y, THREE.RGBAFormat, covariancesTextureType); - covTex.needsUpdate = true; - this.material.uniforms.covariancesTexture.value = covTex; - this.material.uniforms.covariancesTextureSize.value.copy(covTexSize); + this.fillSplatDataArrays(covariances, scales, rotations, centers, colors, shData, undefined, + covarianceCompressionLevel, scaleRotationCompressionLevel, shCompressionLevel); // set up centers/colors data texture const centersColsTexSize = computeDataTextureSize(CENTER_COLORS_ELEMENTS_PER_TEXEL, 4); @@ -652,16 +691,12 @@ export class SplatMesh extends THREE.Mesh { this.splatDataTextures = { 'baseData': { 'covariances': covariances, + 'scales': scales, + 'rotations': rotations, 'centers': centers, 'colors': colors, 'sphericalHarmonics': shData }, - 'covariances': { - 'data': paddedCovariances, - 'texture': covTex, - 'size': covTexSize, - 'compressionLevel': covarianceCompressionLevel - }, 'centerColors': { 'data': paddedCentersCols, 'texture': centersColsTex, @@ -669,6 +704,79 @@ export class SplatMesh extends THREE.Mesh { } }; + if (this.splatRenderMode === SplatRenderMode.ThreeD) { + // set up covariances data texture + + const covTexSpecs = getCovariancesInitialTextureSpecs(covarianceCompressionLevel); + const covariancesElementsPerTexelStored = covTexSpecs.elementsPerTexelStored; + const covTexSize = covTexSpecs.texSize; + + let CovariancesDataType = covarianceCompressionLevel >= 1 ? Uint32Array : Float32Array; + const covariancesElementsPerTexelAllocated = covarianceCompressionLevel >= 1 ? + COVARIANCES_ELEMENTS_PER_TEXEL_COMPRESSED_ALLOCATED : + COVARIANCES_ELEMENTS_PER_TEXEL_ALLOCATED; + const covariancesTextureData = new CovariancesDataType(covTexSize.x * covTexSize.y * covariancesElementsPerTexelAllocated); + + if (covarianceCompressionLevel === 0) { + covariancesTextureData.set(covariances); + } else { + SplatMesh.updatePaddedCompressedCovariancesTextureData(covariances, covariancesTextureData, 0, 0, covariances.length); + } + + let covTex; + if (covarianceCompressionLevel >= 1) { + covTex = new THREE.DataTexture(covariancesTextureData, covTexSize.x, covTexSize.y, + THREE.RGBAIntegerFormat, THREE.UnsignedIntType); + covTex.internalFormat = 'RGBA32UI'; + this.material.uniforms.covariancesTextureHalfFloat.value = covTex; + } else { + covTex = new THREE.DataTexture(covariancesTextureData, covTexSize.x, covTexSize.y, THREE.RGBAFormat, THREE.FloatType); + this.material.uniforms.covariancesTexture.value = covTex; + + // For some reason a usampler2D needs to have a valid texture attached or WebGL complains + const dummyTex = new THREE.DataTexture(new Uint32Array(32), 2, 2, THREE.RGBAIntegerFormat, THREE.UnsignedIntType); + dummyTex.internalFormat = 'RGBA32UI'; + this.material.uniforms.covariancesTextureHalfFloat.value = dummyTex; + dummyTex.needsUpdate = true; + } + covTex.needsUpdate = true; + + this.material.uniforms.covariancesAreHalfFloat.value = (covarianceCompressionLevel >= 1) ? 1 : 0; + this.material.uniforms.covariancesTextureSize.value.copy(covTexSize); + + this.splatDataTextures['covariances'] = { + 'data': covariancesTextureData, + 'texture': covTex, + 'size': covTexSize, + 'compressionLevel': covarianceCompressionLevel, + 'elementsPerTexelStored': covariancesElementsPerTexelStored, + 'elementsPerTexelAllocated': covariancesElementsPerTexelAllocated + }; + } else { + // set up scale & rotations data texture + const elementsPerSplat = 6; + const scaleRotationsTexSize = computeDataTextureSize(SCALES_ROTATIONS_ELEMENTS_PER_TEXEL, elementsPerSplat); + let ScaleRotationsDataType = scaleRotationCompressionLevel >= 1 ? Uint16Array : Float32Array; + let scaleRotationsTextureType = scaleRotationCompressionLevel >= 1 ? THREE.HalfFloatType : THREE.FloatType; + const paddedScaleRotations = new ScaleRotationsDataType(scaleRotationsTexSize.x * scaleRotationsTexSize.y * + SCALES_ROTATIONS_ELEMENTS_PER_TEXEL); + + SplatMesh.updateScaleRotationsPaddedData(0, splatCount - 1, scales, rotations, paddedScaleRotations); + + const scaleRotationsTex = new THREE.DataTexture(paddedScaleRotations, scaleRotationsTexSize.x, scaleRotationsTexSize.y, + THREE.RGBAFormat, scaleRotationsTextureType); + scaleRotationsTex.needsUpdate = true; + this.material.uniforms.scaleRotationsTexture.value = scaleRotationsTex; + this.material.uniforms.scaleRotationsTextureSize.value.copy(scaleRotationsTexSize); + + this.splatDataTextures['scaleRotations'] = { + 'data': paddedScaleRotations, + 'texture': scaleRotationsTex, + 'size': scaleRotationsTexSize, + 'compressionLevel': scaleRotationCompressionLevel + }; + } + if (shData) { const shTextureType = shCompressionLevel === 2 ? THREE.UnsignedByteType : THREE.HalfFloatType; @@ -678,15 +786,8 @@ export class SplatMesh extends THREE.Mesh { const texelFormat = shElementsPerTexel === 4 ? THREE.RGBAFormat : THREE.RGFormat; let shTexSize = computeDataTextureSize(shElementsPerTexel, paddedSHComponentCount); - // Based on my own observations across multiple devices, OSes and browsers, using textures that have one dimension - // greater than 4096 while the other is greater than or equal to 4096 causes issues (Essentially any texture larger - // than 4096 x 4096 (16777216) texels). Specifically it seems all texture data beyond the 4096 x 4096 texel boundary - // is corrupted, while data below that boundary is usable. In these cases the texture has been valid in the eyes of - // both Three.js and WebGL, and the texel format (RG, RGBA, etc.) has not mattered. More investigation will be needed, - // but for now the work-around is to split the spherical harmonics into three textures (one for each color channel). - // Use one texture for all spherical harmonics data - if (shTexSize.x * shTexSize.y <= 16777216) { + if (shTexSize.x * shTexSize.y <= MAX_TEXTURE_TEXELS) { const paddedSHArraySize = shTexSize.x * shTexSize.y * shElementsPerTexel; const paddedSHArray = new SphericalHarmonicsArrayType(paddedSHArraySize); for (let c = 0; c < splatCount; c++) { @@ -783,40 +884,29 @@ export class SplatMesh extends THREE.Mesh { } updateBaseDataFromSplatBuffers(fromSplat, toSplat) { - const covarianceCompressionLevel = this.splatDataTextures['covariances'].compressionLevel; - const shTextureDesc = this.splatDataTextures['sphericalHarmonics']; - const shCompressionLevel = shTextureDesc ? shTextureDesc.compressionLevel : 0; - - this.fillSplatDataArrays(this.splatDataTextures.baseData.covariances, - this.splatDataTextures.baseData.centers, this.splatDataTextures.baseData.colors, - this.splatDataTextures.baseData.sphericalHarmonics, undefined, covarianceCompressionLevel, - shCompressionLevel, fromSplat, toSplat, fromSplat); + const covarancesTextureDesc = this.splatDataTextures['covariances']; + const covarianceCompressionLevel = covarancesTextureDesc ? covarancesTextureDesc.compressionLevel : undefined; + const scaleRotationsTextureDesc = this.splatDataTextures['scaleRotations']; + const scaleRotationCompressionLevel = scaleRotationsTextureDesc ? scaleRotationsTextureDesc.compressionLevel : undefined; + const shITextureDesc = this.splatDataTextures['sphericalHarmonics']; + const shCompressionLevel = shITextureDesc ? shITextureDesc.compressionLevel : 0; + + this.fillSplatDataArrays(this.splatDataTextures.baseData.covariances, this.splatDataTextures.baseData.scales, + this.splatDataTextures.baseData.rotations, this.splatDataTextures.baseData.centers, + this.splatDataTextures.baseData.colors, this.splatDataTextures.baseData.sphericalHarmonics, undefined, + covarianceCompressionLevel, scaleRotationCompressionLevel, shCompressionLevel, + fromSplat, toSplat, fromSplat); } updateDataTexturesFromBaseData(fromSplat, toSplat) { - const covarianceCompressionLevel = this.splatDataTextures['covariances'].compressionLevel; + const covarancesTextureDesc = this.splatDataTextures['covariances']; + const covarianceCompressionLevel = covarancesTextureDesc ? covarancesTextureDesc.compressionLevel : undefined; + const scaleRotationsTextureDesc = this.splatDataTextures['scaleRotations']; + const scaleRotationCompressionLevel = scaleRotationsTextureDesc ? scaleRotationsTextureDesc.compressionLevel : undefined; const shTextureDesc = this.splatDataTextures['sphericalHarmonics']; const shCompressionLevel = shTextureDesc ? shTextureDesc.compressionLevel : 0; - const covariancesTextureDescriptor = this.splatDataTextures['covariances']; - const paddedCovariances = covariancesTextureDescriptor.data; - const covariancesTexture = covariancesTextureDescriptor.texture; - const covarancesStartSplat = fromSplat * COVARIANCES_ELEMENTS_PER_SPLAT; - const covariancesEndSplat = toSplat * COVARIANCES_ELEMENTS_PER_SPLAT; - for (let i = covarancesStartSplat; i <= covariancesEndSplat; i++) { - const covariance = this.splatDataTextures.baseData.covariances[i]; - paddedCovariances[i] = covariance; - } - const covariancesTextureProps = this.renderer ? this.renderer.properties.get(covariancesTexture) : null; - if (!covariancesTextureProps || !covariancesTextureProps.__webglTexture) { - covariancesTexture.needsUpdate = true; - } else { - const covaranceBytesPerElement = covarianceCompressionLevel ? 2 : 4; - this.updateDataTexture(paddedCovariances, covariancesTextureDescriptor.texture, covariancesTextureDescriptor.size, - covariancesTextureProps, COVARIANCES_ELEMENTS_PER_TEXEL, COVARIANCES_ELEMENTS_PER_SPLAT, - covaranceBytesPerElement, fromSplat, toSplat); - } - + // Update center & color data texture const centerColorsTextureDescriptor = this.splatDataTextures['centerColors']; const paddedCenterColors = centerColorsTextureDescriptor.data; const centerColorsTexture = centerColorsTextureDescriptor.texture; @@ -831,6 +921,60 @@ export class SplatMesh extends THREE.Mesh { fromSplat, toSplat); } + // update covariance data texture + if (covarancesTextureDesc) { + const covariancesTexture = covarancesTextureDesc.texture; + const covarancesStartElement = fromSplat * COVARIANCES_ELEMENTS_PER_SPLAT; + const covariancesEndElement = toSplat * COVARIANCES_ELEMENTS_PER_SPLAT; + + if (covarianceCompressionLevel === 0) { + for (let i = covarancesStartElement; i <= covariancesEndElement; i++) { + const covariance = this.splatDataTextures.baseData.covariances[i]; + covarancesTextureDesc.data[i] = covariance; + } + } else { + SplatMesh.updatePaddedCompressedCovariancesTextureData(this.splatDataTextures.baseData.covariances, + covarancesTextureDesc.data, + fromSplat * covarancesTextureDesc.elementsPerTexelAllocated, + covarancesStartElement, covariancesEndElement); + } + + const covariancesTextureProps = this.renderer ? this.renderer.properties.get(covariancesTexture) : null; + if (!covariancesTextureProps || !covariancesTextureProps.__webglTexture) { + covariancesTexture.needsUpdate = true; + } else { + if (covarianceCompressionLevel === 0) { + this.updateDataTexture(covarancesTextureDesc.data, covarancesTextureDesc.texture, covarancesTextureDesc.size, + covariancesTextureProps, covarancesTextureDesc.elementsPerTexelStored, + COVARIANCES_ELEMENTS_PER_SPLAT, 4, fromSplat, toSplat); + } else { + this.updateDataTexture(covarancesTextureDesc.data, covarancesTextureDesc.texture, covarancesTextureDesc.size, + covariancesTextureProps, covarancesTextureDesc.elementsPerTexelAllocated, + covarancesTextureDesc.elementsPerTexelAllocated, 2, fromSplat, toSplat); + } + } + } + + // update scale and rotation data texture + if (scaleRotationsTextureDesc) { + const paddedScaleRotations = scaleRotationsTextureDesc.data; + const scaleRotationsTexture = scaleRotationsTextureDesc.texture; + const elementsPerSplat = 6; + const bytesPerElement = scaleRotationCompressionLevel === 0 ? 4 : 2; + + SplatMesh.updateScaleRotationsPaddedData(fromSplat, toSplat, this.splatDataTextures.baseData.scales, + this.splatDataTextures.baseData.rotations, paddedScaleRotations); + const scaleRotationsTextureProps = this.renderer ? this.renderer.properties.get(scaleRotationsTexture) : null; + if (!scaleRotationsTextureProps || !scaleRotationsTextureProps.__webglTexture) { + scaleRotationsTexture.needsUpdate = true; + } else { + this.updateDataTexture(paddedScaleRotations, scaleRotationsTextureDesc.texture, scaleRotationsTextureDesc.size, + scaleRotationsTextureProps, SCALES_ROTATIONS_ELEMENTS_PER_TEXEL, elementsPerSplat, bytesPerElement, + fromSplat, toSplat); + } + } + + // update spherical harmonics data texture const shData = this.splatDataTextures.baseData.sphericalHarmonics; if (shData) { let shBytesPerElement = 4; @@ -883,6 +1027,7 @@ export class SplatMesh extends THREE.Mesh { } } + // update scene index & transform data if (this.dynamicMode) { const sceneIndexesTexDesc = this.splatDataTextures['sceneIndexes']; const paddedTransformIndexes = sceneIndexesTexDesc.data; @@ -952,12 +1097,12 @@ export class SplatMesh extends THREE.Mesh { }; } - updateDataTexture(paddedData, texture, textureSize, textureProps, elementsPerTexel, elementsPerSplat, bytesPerSplat, from, to) { + updateDataTexture(paddedData, texture, textureSize, textureProps, elementsPerTexel, elementsPerSplat, bytesPerElement, from, to) { const gl = this.renderer.getContext(); const updateRegion = SplatMesh.computeTextureUpdateRegion(from, to, textureSize.x, elementsPerTexel, elementsPerSplat); const updateElementCount = updateRegion.dataEnd - updateRegion.dataStart; const updateDataView = new paddedData.constructor(paddedData.buffer, - updateRegion.dataStart * bytesPerSplat, updateElementCount); + updateRegion.dataStart * bytesPerElement, updateElementCount); const updateHeight = updateRegion.endRow - updateRegion.startRow + 1; const glType = this.webGLUtils.convert(texture.type); const glFormat = this.webGLUtils.convert(texture.format, texture.colorSpace); @@ -968,6 +1113,21 @@ export class SplatMesh extends THREE.Mesh { gl.bindTexture(gl.TEXTURE_2D, currentTexture); } + static updatePaddedCompressedCovariancesTextureData(sourceData, textureData, textureDataStartIndex, fromElement, toElement) { + let textureDataView = new DataView(textureData.buffer); + let textureDataIndex = textureDataStartIndex; + let sequentialCount = 0; + for (let i = fromElement; i <= toElement; i+=2) { + textureDataView.setUint16(textureDataIndex * 2, sourceData[i], true); + textureDataView.setUint16(textureDataIndex * 2 + 2, sourceData[i + 1], true); + textureDataIndex += 2; + sequentialCount++; + if (sequentialCount >= 3) { + textureDataIndex += 2; + sequentialCount = 0; + } + } + } static updateCenterColorsPaddedData(from, to, centers, colors, paddedCenterColors) { for (let c = from; c <= to; c++) { @@ -981,6 +1141,23 @@ export class SplatMesh extends THREE.Mesh { } } + static updateScaleRotationsPaddedData(from, to, scales, rotations, paddedScaleRotations) { + const combinedSize = 6; + for (let c = from; c <= to; c++) { + const scaleBase = c * 3; + const rotationBase = c * 4; + const scaleRotationsBase = c * combinedSize; + + paddedScaleRotations[scaleRotationsBase] = scales[scaleBase]; + paddedScaleRotations[scaleRotationsBase + 1] = scales[scaleBase + 1]; + paddedScaleRotations[scaleRotationsBase + 2] = scales[scaleBase + 2]; + + paddedScaleRotations[scaleRotationsBase + 3] = rotations[rotationBase]; + paddedScaleRotations[scaleRotationsBase + 4] = rotations[rotationBase + 1]; + paddedScaleRotations[scaleRotationsBase + 5] = rotations[rotationBase + 2]; + } + } + updateVisibleRegion(sinceLastBuildOnly) { const splatCount = this.getSplatCount(); const tempCenter = new THREE.Vector3(); @@ -1646,6 +1823,8 @@ export class SplatMesh extends THREE.Mesh { /** * Fill arrays with splat data and apply transforms if appropriate. Each array is optional. * @param {Float32Array} covariances Target storage for splat covariances + * @param {Float32Array} scales Target storage for splat scales + * @param {Float32Array} rotations Target storage for splat rotations * @param {Float32Array} centers Target storage for splat centers * @param {Uint8Array} colors Target storage for splat colors * @param {Float32Array} sphericalHarmonics Target storage for spherical harmonics @@ -1658,8 +1837,17 @@ export class SplatMesh extends THREE.Mesh { * @param {number} srcEnd The end location from which to pull source data * @param {number} destStart The start location from which to write data */ - fillSplatDataArrays(covariances, centers, colors, sphericalHarmonics, applySceneTransform, - covarianceCompressionLevel = 0, sphericalHarmonicsCompressionLevel = 1, srcStart, srcEnd, destStart = 0) { + fillSplatDataArrays(covariances, scales, rotations, centers, colors, sphericalHarmonics, applySceneTransform, + covarianceCompressionLevel = 0, scaleRotationCompressionLevel = 0, sphericalHarmonicsCompressionLevel = 1, + srcStart, srcEnd, destStart = 0) { + const scaleOverride = new THREE.Vector3(); + scaleOverride.x = undefined; + scaleOverride.y = undefined; + if (this.splatRenderMode === SplatRenderMode.ThreeD) { + scaleOverride.z = undefined; + } else { + scaleOverride.z = 1; + } for (let i = 0; i < this.scenes.length; i++) { if (applySceneTransform === undefined || applySceneTransform === null) { @@ -1670,8 +1858,14 @@ export class SplatMesh extends THREE.Mesh { const splatBuffer = scene.splatBuffer; const sceneTransform = applySceneTransform ? scene.transform : null; if (covariances) { - splatBuffer.fillSplatCovarianceArray(covariances, sceneTransform, - srcStart, srcEnd, destStart, covarianceCompressionLevel); + splatBuffer.fillSplatCovarianceArray(covariances, sceneTransform, srcStart, srcEnd, destStart, covarianceCompressionLevel); + } + if (scales || rotations) { + if (!scales || !rotations) { + throw new Error('SplatMesh::fillSplatDataArrays() -> "scales" and "rotations" must both be valid.'); + } + splatBuffer.fillSplatScaleRotationArray(scales, rotations, sceneTransform, + srcStart, srcEnd, destStart, scaleRotationCompressionLevel, scaleOverride); } if (centers) splatBuffer.fillSplatCenterArray(centers, sceneTransform, srcStart, srcEnd, destStart); if (colors) splatBuffer.fillSplatColorArray(colors, scene.minimumAlpha, srcStart, srcEnd, destStart); @@ -1694,7 +1888,7 @@ export class SplatMesh extends THREE.Mesh { getIntegerCenters(start, end, padFour = false) { const splatCount = end - start + 1; const floatCenters = new Float32Array(splatCount * 3); - this.fillSplatDataArrays(null, floatCenters, null, null, undefined, undefined, undefined, start); + this.fillSplatDataArrays(null, null, null, floatCenters, null, null, undefined, undefined, undefined, undefined, start); let intCenters; let componentCount = padFour ? 4 : 3; intCenters = new Int32Array(splatCount * componentCount); @@ -1717,7 +1911,7 @@ export class SplatMesh extends THREE.Mesh { getFloatCenters(start, end, padFour = false) { const splatCount = end - start + 1; const floatCenters = new Float32Array(splatCount * 3); - this.fillSplatDataArrays(null, floatCenters, null, null, undefined, undefined, undefined, start); + this.fillSplatDataArrays(null, null, null, floatCenters, null, null, undefined, undefined, undefined, undefined, start); if (!padFour) return floatCenters; let paddedFloatCenters = new Float32Array(splatCount * 4); for (let i = 0; i < splatCount; i++) { @@ -1763,10 +1957,16 @@ export class SplatMesh extends THREE.Mesh { getSplatScaleAndRotation = function() { const paramsObj = {}; + const scaleOverride = new THREE.Vector3(); return function(globalIndex, outScale, outRotation, applySceneTransform) { this.getLocalSplatParameters(globalIndex, paramsObj, applySceneTransform); - paramsObj.splatBuffer.getSplatScaleAndRotation(paramsObj.localIndex, outScale, outRotation, paramsObj.sceneTransform); + scaleOverride.x = undefined; + scaleOverride.y = undefined; + scaleOverride.z = undefined; + if (this.splatRenderMode === SplatRenderMode.TwoD) scaleOverride.z = 0; + paramsObj.splatBuffer.getSplatScaleAndRotation(paramsObj.localIndex, outScale, outRotation, + paramsObj.sceneTransform, scaleOverride); }; }();