Gaussian Splatting
3D Gaussian Splatting is a rasterization technique that allows to render 3D objects with a high level of detail. It’s currently quickly gaining popularity because of the photorealistic rendering of 3D scan data. This example shows how to implement 3D Gaussian Splatting with the help of two libraries:
The technology presented in this example is rapidly developing and therefore the components are
not part of the @threlte/extras package as of now.
You may however copy and paste them into your project and use them as you wish.
<script>
import { Canvas } from '@threlte/core'
import Scene from './Scene.svelte'
</script>
<div>
<Canvas>
<Scene />
</Canvas>
</div>
<style>
:global(body) {
margin: 0;
}
div {
width: 100%;
height: 100%;
}
</style><script lang="ts">
import { LumaSplatsThree, type LumaSplatsSource } from '@lumaai/luma-web'
import { T, asyncWritable, useCache, useTask, useThrelte } from '@threlte/core'
import { useSuspense } from '@threlte/extras'
import { onDestroy } from 'svelte'
import type { CubeTexture, Scene } from 'three'
export let source: LumaSplatsSource
export let mode: 'object' | 'object-env' | 'env' = 'object'
export let loadingAnimationEnabled = false
export let particleRevealEnabled = false
export let enableThreeShaderIntegration = true
const { invalidate, renderer, scene } = useThrelte()
const { remember } = useCache()
const suspend = useSuspense()
const captureCubemap = mode === 'env' || mode === 'object-env'
const splats = suspend(
asyncWritable(
remember(() => {
return new Promise<[LumaSplatsThree, CubeTexture | undefined]>((resolve) => {
const splats = new LumaSplatsThree({
source,
loadingAnimationEnabled,
particleRevealEnabled,
enableThreeShaderIntegration
})
splats.onLoad = async () => {
if (captureCubemap) {
splats.captureCubemap(renderer).then((cubemap) => {
resolve([splats, cubemap])
})
} else {
resolve([splats, undefined])
}
}
})
}, [source])
)
)
let preheat =
particleRevealEnabled && loadingAnimationEnabled ? 400 : loadingAnimationEnabled ? 100 : 10
let frame = 0
const { start, stop } = useTask(
() => {
frame++
if (frame >= preheat) {
stop()
frame = 0
}
},
{ autoStart: false }
)
let previousEnvironment: Scene['environment']
let previousBackground: Scene['background']
let previousBackgroundBluriness: Scene['backgroundBlurriness']
$: if ($splats && $splats[1]) {
previousEnvironment = scene.environment
previousBackground = scene.background
previousBackgroundBluriness = scene.backgroundBlurriness
scene.environment = $splats[1]
scene.background = $splats[1]
scene.backgroundBlurriness = 0.5
invalidate()
}
onDestroy(() => {
if (captureCubemap) {
scene.environment = previousEnvironment
scene.background = previousBackground
scene.backgroundBlurriness = previousBackgroundBluriness
invalidate()
}
})
</script>
{#if (mode === 'object' || mode === 'object-env') && $splats && $splats[0]}
<T
dispose={false}
is={$splats[0]}
oncreate={() => {
start()
}}
{...$$restProps}
>
<slot ref={$splats[0]} />
</T>
{/if}import type { Events, Props, Slots } from '@threlte/core'
import { SvelteComponent } from 'svelte'
import { LumaSplatsSemantics, LumaSplatsThree, type LumaSplatsSource } from '@lumaai/luma-web'
export type LumaSplatsThreeProps = Props<LumaSplatsThree> & {
source: LumaSplatsSource
mode?: 'object' | 'env' | 'object-env'
loadingAnimationEnabled?: boolean
particleRevealEnabled?: boolean
enableThreeShaderIntegration?: boolean
}
export type LumaSplatsThreeEvents = Events<LumaSplatsThree>
export type LumaSplatsThreeSlots = Slots<LumaSplatsThree>
export default class LumaSplatsThree extends SvelteComponent<
LumaSplatsThreeProps,
LumaSplatsThreeEvents,
LumaSplatsThreeSlots
> {}<script lang="ts">
import { useStage, useTask, useThrelte } from '@threlte/core'
import { WaveformMonitor } from 'svelte-tweakpane-ui'
const { shouldRender, renderStage } = useThrelte()
const afterRenderStage = useStage('after-render', {
after: renderStage
})
let log = Array(100).fill(0)
useTask(
() => {
log = update(log)
},
{
autoInvalidate: false,
stage: afterRenderStage
}
)
function update(log: number[]) {
log.shift()
log.push(shouldRender() ? 1 : 0)
return log
}
</script>
<WaveformMonitor
value={log}
min={-1}
max={2}
/><script lang="ts">
import { T } from '@threlte/core'
import { GLTF, OrbitControls } from '@threlte/extras'
import { Checkbox, Folder, FpsGraph, List, Pane, Slider } from 'svelte-tweakpane-ui'
import type { MeshStandardMaterial } from 'three'
import { DEG2RAD } from 'three/src/math/MathUtils.js'
import LumaSplats from './LumaSplatsThree/LumaSplatsThree.svelte'
import RenderIndicator from './RenderIndicator.svelte'
import Splat from './Splat/Splat.svelte'
// <LumaSplatsThree>
let showLumaSplats = true
let lumaSplatsMode: 'object' | 'object-env' | 'env' = 'object-env'
// <Splat>
let showSplat = true
let alphaHash = false
let alphaTest = 0.06
let toneMapped = true
// Car
let showPorsche = true
let gltfMaterials: Record<string, MeshStandardMaterial> | undefined
$: if (gltfMaterials) {
Object.values(gltfMaterials).forEach((material) => {
material.envMapIntensity = 5
})
}
</script>
<LumaSplats
visible={showLumaSplats}
source="https://lumalabs.ai/capture/4c15c22e-8655-4423-aeac-b08f017dda22"
mode={lumaSplatsMode}
/>
<Splat
visible={showSplat}
position={[1.08, 2.21, -1.99]}
rotation={[-32.3 * DEG2RAD, -18.5 * DEG2RAD, -6.4 * DEG2RAD]}
src="https://huggingface.co/cakewalk/splat-data/resolve/main/nike.splat"
{alphaHash}
alphaTest={alphaTest > 0 ? alphaTest : undefined}
{toneMapped}
/>
<GLTF
visible={showPorsche}
position={[-1.48, -0.51, 2.15]}
rotation.y={57 * DEG2RAD}
scale={0.7}
bind:materials={gltfMaterials}
url="/models/splat-example/porsche_959.glb"
/>
<T.PerspectiveCamera
makeDefault
position={[0.22, 2.44, 9.06]}
oncreate={({ ref }) => {
ref.lookAt(0, 0, 0)
}}
fov={25}
>
<OrbitControls />
</T.PerspectiveCamera>
<Pane
position="fixed"
title="Gaussian Splatting"
>
<Folder
userExpandable={false}
expanded
title="Luma"
>
<Checkbox
bind:value={showLumaSplats}
label="Show LumaSplats"
/>
{#if showLumaSplats}
<List
bind:value={lumaSplatsMode}
options={{
object: 'object',
'object-env': 'object-env',
env: 'env'
}}
/>
{/if}
</Folder>
<Folder
userExpandable={false}
expanded
title="Splat"
>
<Checkbox
bind:value={showSplat}
label="Show Splats"
/>
{#if showSplat}
<Checkbox
bind:value={alphaHash}
label="alphaHash"
/>
<Slider
bind:value={alphaTest}
label="alphaTest"
min={0}
max={1}
step={0.01}
/>
<Checkbox
bind:value={toneMapped}
label="toneMapped"
/>
{/if}
</Folder>
<Folder title="Porsche">
<Checkbox
bind:value={showPorsche}
label="Show Porsche"
/>
</Folder>
<Folder title="Rendering Activity">
<RenderIndicator />
<FpsGraph />
</Folder>
</Pane><script lang="ts">
import { Splat, SplatLoader } from '@pmndrs/vanilla'
import { T, useLoader, useTask, useThrelte } from '@threlte/core'
export let src: string
export let alphaHash = false
export let alphaTest: number | undefined = undefined
export let toneMapped: boolean | undefined = undefined
const { renderer, camera } = useThrelte()
const loader = useLoader(SplatLoader, {
args: [renderer]
})
let framesRendered = 0
const { start, stop } = useTask(
() => {
framesRendered++
// render for 10 frames
if (framesRendered >= 10) {
stop()
framesRendered = 0
}
},
{ autoStart: false }
)
</script>
{#await loader.load(src) then splat}
<T
{...$$restProps}
dispose={false}
is={Splat}
args={[
splat,
$camera,
{
alphaHash,
alphaTest,
toneMapped
}
]}
oncreate={start}
>
<slot ref={Splat} />
</T>
{/await}import type { Splat } from '@pmndrs/vanilla'
import type { Events, Props, Slots } from '@threlte/core'
import { SvelteComponent } from 'svelte'
export type SplatProps = Props<Splat> & {
alphaHash?: boolean
alphaTest?: number | undefined
toneMapped?: boolean
}
export type SplatEvents = Events<Splat>
export type SplatSlots = Slots<Splat>
export default class Splat extends SvelteComponent<SplatProps, SplatEvents, SplatSlots> {}Models: