diff --git a/package-lock.json b/package-lock.json index eae03d5..a9be2ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "super-splat", - "version": "0.10.4", + "version": "0.11.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "super-splat", - "version": "0.10.4", + "version": "0.11.0", "license": "MIT", "devDependencies": { "@playcanvas/eslint-config": "^1.4.1", diff --git a/package.json b/package.json index 1360ea7..f084517 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "super-splat", - "version": "0.10.4", + "version": "0.11.0", "author": "PlayCanvas", "homepage": "https://playcanvas.com", "description": "All the splat things", diff --git a/src/asset-loader.ts b/src/asset-loader.ts index a7b6904..a5d505d 100644 --- a/src/asset-loader.ts +++ b/src/asset-loader.ts @@ -4,7 +4,7 @@ import { Model } from './model'; import { Splat } from './splat'; import { Env } from './env'; -import { startSpinner, stopSpinner } from './spinner'; +import { startSpinner, stopSpinner } from './ui/spinner'; interface ModelLoadRequest { url?: string; @@ -67,15 +67,15 @@ class AssetLoader { } as any ); containerAsset.on('load', () => { + stopSpinner(); if (isPly) { resolve(new Splat(containerAsset)); } else { resolve(new Model(containerAsset, gemMaterials)); } - - stopSpinner(); }); containerAsset.on('error', (err: string) => { + stopSpinner(); reject(err); }); diff --git a/src/editor-ops.ts b/src/editor-ops.ts index 8f656e1..d92a213 100644 --- a/src/editor-ops.ts +++ b/src/editor-ops.ts @@ -1,128 +1,21 @@ import { - BLEND_NORMAL, BoundingBox, Color, Mat4, - Material, - Mesh, - MeshInstance, path, - PRIMITIVE_POINTS, - Quat, - SEMANTIC_POSITION, - createShaderFromCode, Vec3, Vec4 } from 'playcanvas'; import { Splat as SplatRender, SplatData, SplatInstance } from 'playcanvas-extras'; import { Scene } from './scene'; -import { EditorUI } from './editor-ui'; +import { EditorUI } from './ui/editor'; import { Element, ElementType } from './element'; import { Splat } from './splat'; import { EditHistory } from './edit-history'; import { deletedOpacity, DeleteSelectionEditOp, ResetEditOp } from './edit-ops'; - -const vs = /* glsl */ ` -attribute vec4 vertex_position; - -uniform mat4 matrix_model; -uniform mat4 matrix_view; -uniform mat4 matrix_projection; -uniform mat4 matrix_viewProjection; - -uniform float splatSize; - -varying vec4 color; - -void main(void) { - if (vertex_position.w == -1.0) { - gl_Position = vec4(0.0, 0.0, 2.0, 1.0); - } else { - gl_Position = matrix_viewProjection * matrix_model * vec4(vertex_position.xyz, 1.0); - gl_PointSize = splatSize; - float opacity = vertex_position.w; - color = (opacity == -1.0) ? vec4(0) : mix(vec4(0, 0, 1.0, 0.5), vec4(1.0, 1.0, 0.0, 0.5), opacity); - } -} -`; - -const fs = /* glsl */ ` -varying vec4 color; -void main(void) -{ - gl_FragColor = color; -} -`; - -class SplatDebug { - splatData: SplatData; - meshInstance: MeshInstance; - - constructor(scene: Scene, splat: Splat, splatData: SplatData) { - const device = scene.graphicsDevice; - - const shader = createShaderFromCode(device, vs, fs, `splatDebugShader`, { - vertex_position: SEMANTIC_POSITION - }); - - const material = new Material(); - material.name = 'splatDebugMaterial'; - material.blendType = BLEND_NORMAL; - material.shader = shader; - material.setParameter('splatSize', 1.0); - material.update(); - - const x = splatData.getProp('x'); - const y = splatData.getProp('y'); - const z = splatData.getProp('z'); - const s = splatData.getProp('selection'); - - const vertexData = new Float32Array(splatData.numSplats * 4); - for (let i = 0; i < splatData.numSplats; ++i) { - vertexData[i * 4 + 0] = x[i]; - vertexData[i * 4 + 1] = y[i]; - vertexData[i * 4 + 2] = z[i]; - vertexData[i * 4 + 3] = s[i]; - } - - const mesh = new Mesh(device); - mesh.setPositions(vertexData, 4); - mesh.update(PRIMITIVE_POINTS, true); - - this.splatData = splatData; - this.meshInstance = new MeshInstance(mesh, material, splat.root); - } - - update() { - const splatData = this.splatData; - const s = splatData.getProp('selection'); - const o = splatData.getProp('opacity'); - - const vb = this.meshInstance.mesh.vertexBuffer; - const vertexData = new Float32Array(vb.lock()); - - let count = 0; - - for (let i = 0; i < splatData.numSplats; ++i) { - const selection = o[i] === deletedOpacity ? -1 : s[i]; - vertexData[i * 4 + 3] = selection; - count += selection === 1 ? 1 : 0; - } - - vb.unlock(); - - return count; - } - - set splatSize(splatSize: number) { - this.meshInstance.material.setParameter('splatSize', splatSize); - } - - get splatSize() { - // @ts-ignore - return this.meshInstance.material.getParameter('splatSize').data; - } -} +import { SplatDebug } from './splat-debug'; +import { convertPly, convertPlyCompressed, convertSplat } from './splat-convert'; +import { startSpinner, stopSpinner } from './ui/spinner'; // download the data uri const download = (filename: string, data: ArrayBuffer) => { @@ -148,171 +41,6 @@ const download = (filename: string, data: ArrayBuffer) => { window.URL.revokeObjectURL(url); }; -const convertPly = (splatData: SplatData, modelMat: Mat4) => { - // count the number of non-deleted splats - const opacity = splatData.getProp('opacity'); - let numSplats = 0; - for (let i = 0; i < splatData.numSplats; ++i) { - numSplats += opacity[i] !== deletedOpacity ? 1 : 0; - } - - const internalProps = ['selection', 'opacityOrig']; - const props = splatData.vertexElement.properties.filter(p => p.storage && !internalProps.includes(p.name)).map(p => p.name); - const header = (new TextEncoder()).encode(`ply\nformat binary_little_endian 1.0\nelement vertex ${numSplats}\n` + props.map(p => `property float ${p}`).join('\n') + `\nend_header\n`); - const result = new Uint8Array(header.byteLength + numSplats * props.length * 4); - - result.set(header); - - const dataView = new DataView(result.buffer); - let offset = header.byteLength; - - for (let i = 0; i < splatData.numSplats; ++i) { - props.forEach((prop) => { - const p = splatData.getProp(prop); - if (p) { - if (opacity[i] !== deletedOpacity) { - dataView.setFloat32(offset, p[i], true); - offset += 4; - } - } - }); - } - - // FIXME - // we must undo the transform we apply at load time to output data - const mat = new Mat4(); - mat.setScale(-1, -1, 1); - mat.invert(); - mat.mul2(mat, modelMat); - - const quat = new Quat(); - quat.setFromMat4(mat); - - const scale = new Vec3(); - mat.getScale(scale); - - const v = new Vec3(); - const q = new Quat(); - - const x_off = props.indexOf('x') * 4; - const y_off = props.indexOf('y') * 4; - const z_off = props.indexOf('z') * 4; - const r0_off = props.indexOf('rot_0') * 4; - const r1_off = props.indexOf('rot_1') * 4; - const r2_off = props.indexOf('rot_2') * 4; - const r3_off = props.indexOf('rot_3') * 4; - const scale0_off = props.indexOf('scale_0') * 4; - const scale1_off = props.indexOf('scale_1') * 4; - const scale2_off = props.indexOf('scale_2') * 4; - - for (let i = 0; i < numSplats; ++i) { - const off = header.byteLength + i * props.length * 4; - const x = dataView.getFloat32(off + x_off, true); - const y = dataView.getFloat32(off + y_off, true); - const z = dataView.getFloat32(off + z_off, true); - const rot_0 = dataView.getFloat32(off + r0_off, true); - const rot_1 = dataView.getFloat32(off + r1_off, true); - const rot_2 = dataView.getFloat32(off + r2_off, true); - const rot_3 = dataView.getFloat32(off + r3_off, true); - const scale_0 = dataView.getFloat32(off + scale0_off, true); - const scale_1 = dataView.getFloat32(off + scale1_off, true); - const scale_2 = dataView.getFloat32(off + scale2_off, true); - - v.set(x, y, z); - mat.transformPoint(v, v); - dataView.setFloat32(off + x_off, v.x, true); - dataView.setFloat32(off + y_off, v.y, true); - dataView.setFloat32(off + z_off, v.z, true); - - q.set(rot_1, rot_2, rot_3, rot_0).mul2(quat, q); - dataView.setFloat32(off + r0_off, q.w, true); - dataView.setFloat32(off + r1_off, q.x, true); - dataView.setFloat32(off + r2_off, q.y, true); - dataView.setFloat32(off + r3_off, q.z, true); - - dataView.setFloat32(off + scale0_off, Math.log(Math.exp(scale_0) * scale.x), true); - dataView.setFloat32(off + scale1_off, Math.log(Math.exp(scale_1) * scale.x), true); - dataView.setFloat32(off + scale2_off, Math.log(Math.exp(scale_2) * scale.x), true); - } - - return result; -}; - -const convertSplat = (splatData: SplatData, modelMat: Mat4) => { - // count the number of non-deleted splats - const x = splatData.getProp('x'); - const y = splatData.getProp('y'); - const z = splatData.getProp('z'); - const opacity = splatData.getProp('opacity'); - const rot_0 = splatData.getProp('rot_0'); - const rot_1 = splatData.getProp('rot_1'); - const rot_2 = splatData.getProp('rot_2'); - const rot_3 = splatData.getProp('rot_3'); - const f_dc_0 = splatData.getProp('f_dc_0'); - const f_dc_1 = splatData.getProp('f_dc_1'); - const f_dc_2 = splatData.getProp('f_dc_2'); - const scale_0 = splatData.getProp('scale_0'); - const scale_1 = splatData.getProp('scale_1'); - const scale_2 = splatData.getProp('scale_2'); - - // count number of non-deleted splats - let numSplats = 0; - for (let i = 0; i < splatData.numSplats; ++i) { - numSplats += opacity[i] !== deletedOpacity ? 1 : 0; - } - - // position.xyz: float32, scale.xyz: float32, color.rgba: uint8, quaternion.ijkl: uint8 - const result = new Uint8Array(numSplats * 32); - const dataView = new DataView(result.buffer); - - // we must undo the transform we apply at load time to output data - const mat = new Mat4(); - mat.setScale(-1, -1, 1); - mat.invert(); - mat.mul2(mat, modelMat); - - const quat = new Quat(); - quat.setFromMat4(mat); - - const v = new Vec3(); - const q = new Quat(); - - const scale = new Vec3(); - mat.getScale(scale); - - const clamp = (x: number) => Math.max(0, Math.min(255, x)); - let idx = 0; - - for (let i = 0; i < splatData.numSplats; ++i) { - if (opacity[i] === deletedOpacity) continue; - - const off = idx++ * 32; - - v.set(x[i], y[i], z[i]); - mat.transformPoint(v, v); - dataView.setFloat32(off + 0, v.x, true); - dataView.setFloat32(off + 4, v.y, true); - dataView.setFloat32(off + 8, v.z, true); - - dataView.setFloat32(off + 12, Math.exp(scale_0[i]) * scale.x, true); - dataView.setFloat32(off + 16, Math.exp(scale_1[i]) * scale.x, true); - dataView.setFloat32(off + 20, Math.exp(scale_2[i]) * scale.x, true); - - const SH_C0 = 0.28209479177387814; - dataView.setUint8(off + 24, clamp((0.5 + SH_C0 * f_dc_0[i]) * 255)); - dataView.setUint8(off + 25, clamp((0.5 + SH_C0 * f_dc_1[i]) * 255)); - dataView.setUint8(off + 26, clamp((0.5 + SH_C0 * f_dc_2[i]) * 255)); - dataView.setUint8(off + 27, clamp((1 / (1 + Math.exp(-opacity[i]))) * 255)); - - q.set(rot_1[i], rot_2[i], rot_3[i], rot_0[i]).mul2(quat, q).normalize(); - dataView.setUint8(off + 28, clamp(q.w * 128 + 128)); - dataView.setUint8(off + 29, clamp(q.x * 128 + 128)); - dataView.setUint8(off + 30, clamp(q.y * 128 + 128)); - dataView.setUint8(off + 31, clamp(q.z * 128 + 128)); - } - - return result; -}; interface SplatDef { element: Splat, @@ -331,6 +59,10 @@ const registerEvents = (scene: Scene, editorUI: EditorUI) => { const aabb = new BoundingBox(); const splatDefs: SplatDef[] = []; + scene.on('error', (err: any) => { + editorUI.showError(err); + }); + // make a copy of the opacity channel because that's what we'll be modifying scene.on('element:added', (element: Element) => { if (element.type === ElementType.splat) { @@ -754,25 +486,39 @@ const registerEvents = (scene: Scene, editorUI: EditorUI) => { scene.assetLoader.loadAllData = value; }); - const removeExtension = (filename: string) => { - return filename.substring(0, filename.length - path.getExtension(filename).length); - }; - events.on('export', (format: string) => { - switch (format) { - case 'ply': - splatDefs.forEach((splatDef) => { - const data = convertPly(splatDef.data, splatDef.element.entity.getWorldTransform()); - download(removeExtension(splatDef.element.asset.file.filename) + '.cleaned.ply', data); - }); - break; - case 'splat': - splatDefs.forEach((splatDef) => { - const data = convertSplat(splatDef.data, splatDef.element.entity.getWorldTransform()); - download(removeExtension(splatDef.element.asset.file.filename) + '.cleaned.splat', data); - }); - break; - } + const removeExtension = (filename: string) => { + return filename.substring(0, filename.length - path.getExtension(filename).length); + }; + + startSpinner(); + editorUI.showInfo('Exporting...'); + + // setTimeout so spinner has a chance to activate + setTimeout(() => { + splatDefs.forEach((splatDef) => { + let data; + let extension; + switch (format) { + case 'ply': + data = convertPly(splatDef.data, splatDef.element.entity.getWorldTransform()); + extension = '.cleaned.ply'; + break; + case 'ply-compressed': + data = convertPlyCompressed(splatDef.data, splatDef.element.entity.getWorldTransform()); + extension = '.compressed.ply'; + break; + case 'splat': + data = convertSplat(splatDef.data, splatDef.element.entity.getWorldTransform()); + extension = '.splat'; + break; + } + download(`${removeExtension(splatDef.element.asset.file.filename)}${extension}`, data); + }); + + stopSpinner(); + editorUI.showInfo(null); + }); }); events.on('undo', () => { diff --git a/src/main.ts b/src/main.ts index 629f36f..fa28479 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,7 +3,7 @@ import { Scene } from './scene'; import { getSceneConfig } from './scene-config'; import { CreateDropHandler } from './drop-handler'; import { initMaterials } from './material'; -import { EditorUI } from './editor-ui'; +import { EditorUI } from './ui/editor'; import { registerEvents } from './editor-ops'; declare global { diff --git a/src/scene.ts b/src/scene.ts index f24c0b7..510944e 100644 --- a/src/scene.ts +++ b/src/scene.ts @@ -229,10 +229,17 @@ class Scene extends EventHandler { } async loadModel(url: string, filename: string) { - const model = await this.assetLoader.loadModel({ url, filename }); - this.add(model); - this.updateBound(); - this.camera.focus(); + // clear error + this.fire('error', null); + + try { + const model = await this.assetLoader.loadModel({ url, filename }); + this.add(model); + this.updateBound(); + this.camera.focus(); + } catch (err) { + this.fire('error', err); + } } clear() { diff --git a/src/splat-convert.ts b/src/splat-convert.ts new file mode 100644 index 0000000..df6e31f --- /dev/null +++ b/src/splat-convert.ts @@ -0,0 +1,498 @@ +import { + Mat4, + Quat, + Vec3 +} from 'playcanvas'; +import { SplatData } from 'playcanvas-extras'; +import { deletedOpacity } from './edit-ops'; + +const convertPly = (splatData: SplatData, modelMat: Mat4) => { + // count the number of non-deleted splats + const opacity = splatData.getProp('opacity'); + let numSplats = 0; + for (let i = 0; i < splatData.numSplats; ++i) { + numSplats += opacity[i] !== deletedOpacity ? 1 : 0; + } + + const internalProps = ['selection', 'opacityOrig']; + const props = splatData.vertexElement.properties.filter(p => p.storage && !internalProps.includes(p.name)).map(p => p.name); + const header = (new TextEncoder()).encode(`ply\nformat binary_little_endian 1.0\nelement vertex ${numSplats}\n` + props.map(p => `property float ${p}`).join('\n') + `\nend_header\n`); + const result = new Uint8Array(header.byteLength + numSplats * props.length * 4); + + result.set(header); + + const dataView = new DataView(result.buffer); + let offset = header.byteLength; + + for (let i = 0; i < splatData.numSplats; ++i) { + props.forEach((prop) => { + const p = splatData.getProp(prop); + if (p) { + if (opacity[i] !== deletedOpacity) { + dataView.setFloat32(offset, p[i], true); + offset += 4; + } + } + }); + } + + // FIXME + // we must undo the transform we apply at load time to output data + const mat = new Mat4(); + mat.setScale(-1, -1, 1); + mat.invert(); + mat.mul2(mat, modelMat); + + const quat = new Quat(); + quat.setFromMat4(mat); + + const scale = new Vec3(); + mat.getScale(scale); + + const v = new Vec3(); + const q = new Quat(); + + const x_off = props.indexOf('x') * 4; + const y_off = props.indexOf('y') * 4; + const z_off = props.indexOf('z') * 4; + const r0_off = props.indexOf('rot_0') * 4; + const r1_off = props.indexOf('rot_1') * 4; + const r2_off = props.indexOf('rot_2') * 4; + const r3_off = props.indexOf('rot_3') * 4; + const scale0_off = props.indexOf('scale_0') * 4; + const scale1_off = props.indexOf('scale_1') * 4; + const scale2_off = props.indexOf('scale_2') * 4; + + for (let i = 0; i < numSplats; ++i) { + const off = header.byteLength + i * props.length * 4; + const x = dataView.getFloat32(off + x_off, true); + const y = dataView.getFloat32(off + y_off, true); + const z = dataView.getFloat32(off + z_off, true); + const rot_0 = dataView.getFloat32(off + r0_off, true); + const rot_1 = dataView.getFloat32(off + r1_off, true); + const rot_2 = dataView.getFloat32(off + r2_off, true); + const rot_3 = dataView.getFloat32(off + r3_off, true); + const scale_0 = dataView.getFloat32(off + scale0_off, true); + const scale_1 = dataView.getFloat32(off + scale1_off, true); + const scale_2 = dataView.getFloat32(off + scale2_off, true); + + v.set(x, y, z); + mat.transformPoint(v, v); + dataView.setFloat32(off + x_off, v.x, true); + dataView.setFloat32(off + y_off, v.y, true); + dataView.setFloat32(off + z_off, v.z, true); + + q.set(rot_1, rot_2, rot_3, rot_0).mul2(quat, q); + dataView.setFloat32(off + r0_off, q.w, true); + dataView.setFloat32(off + r1_off, q.x, true); + dataView.setFloat32(off + r2_off, q.y, true); + dataView.setFloat32(off + r3_off, q.z, true); + + dataView.setFloat32(off + scale0_off, Math.log(Math.exp(scale_0) * scale.x), true); + dataView.setFloat32(off + scale1_off, Math.log(Math.exp(scale_1) * scale.x), true); + dataView.setFloat32(off + scale2_off, Math.log(Math.exp(scale_2) * scale.x), true); + } + + return result; +}; + +const calcMinMax = (data: Float32Array, indices?: number[]) => { + let min; + let max; + if (indices) { + min = max = data[indices[0]]; + for (let i = 1; i < indices.length; ++i) { + const v = data[indices[i]]; + min = Math.min(min, v); + max = Math.max(max, v); + } + } else { + min = max = data[0]; + for (let i = 1; i < data.length; ++i) { + const v = data[i]; + min = Math.min(min, v); + max = Math.max(max, v); + } + } + return { min, max }; +}; + +const quat = new Quat(); +const scale = new Vec3(); +const v = new Vec3(); +const q = new Quat(); + +// process and compress a chunk of 256 splats +class Chunk { + static members = [ + 'x', 'y', 'z', 'scale_0', 'scale_1', 'scale_2', 'f_dc_0', 'f_dc_1', 'f_dc_2', 'opacity', 'rot_0', 'rot_1', 'rot_2', 'rot_3' + ]; + + size: number; + data: any = {}; + + // compressed data + position: Uint32Array; + rotation: Uint32Array; + scale: Uint32Array; + color: Uint32Array; + + constructor(size = 256) { + this.size = size; + Chunk.members.forEach((m) => { + this.data[m] = new Float32Array(size); + }); + this.position = new Uint32Array(size); + this.rotation = new Uint32Array(size); + this.scale = new Uint32Array(size); + this.color = new Uint32Array(size); + } + + set(splatData: SplatData, indices: number[]) { + Chunk.members.forEach((name) => { + const prop = splatData.getProp(name); + const m = this.data[name]; + indices.forEach((idx, i) => { + m[i] = prop[idx]; + }); + }); + } + + transform(mat: Mat4) { + quat.setFromMat4(mat); + mat.getScale(scale); + + const data = this.data; + + const x = data.x; + const y = data.y; + const z = data.z; + const scale_0 = data.scale_0; + const scale_1 = data.scale_1; + const scale_2 = data.scale_2; + const rot_0 = data.rot_0; + const rot_1 = data.rot_1; + const rot_2 = data.rot_2; + const rot_3 = data.rot_3; + + for (let i = 0; i < this.size; ++i) { + // position + v.set(x[i], y[i], z[i]); + mat.transformPoint(v, v); + x[i] = v.x; + y[i] = v.y; + z[i] = v.z; + + // rotation + q.set(rot_1[i], rot_2[i], rot_3[i], rot_0[i]).mul2(quat, q); + rot_0[i] = q.w; + rot_1[i] = q.x; + rot_2[i] = q.y; + rot_3[i] = q.z; + + // scale + scale_0[i] = Math.log(Math.exp(scale_0[i]) * scale.x); + scale_1[i] = Math.log(Math.exp(scale_1[i]) * scale.y); + scale_2[i] = Math.log(Math.exp(scale_2[i]) * scale.z); + } + } + + pack() { + const data = this.data; + + const x = data.x; + const y = data.y; + const z = data.z; + const scale_0 = data.scale_0; + const scale_1 = data.scale_1; + const scale_2 = data.scale_2; + const rot_0 = data.rot_0; + const rot_1 = data.rot_1; + const rot_2 = data.rot_2; + const rot_3 = data.rot_3; + const f_dc_0 = data.f_dc_0; + const f_dc_1 = data.f_dc_1; + const f_dc_2 = data.f_dc_2; + const opacity = data.opacity; + + const px = calcMinMax(x); + const py = calcMinMax(y); + const pz = calcMinMax(z); + + const sx = calcMinMax(scale_0); + const sy = calcMinMax(scale_1); + const sz = calcMinMax(scale_2); + + const packUnorm = (value: number, bits: number) => { + const t = (1 << bits) - 1; + return Math.max(0, Math.min(t, Math.floor(value * t + 0.5))); + }; + + const pack111011 = (x: number, y: number, z: number) => { + return packUnorm(x, 11) << 21 | + packUnorm(y, 10) << 11 | + packUnorm(z, 11); + }; + + const pack8888 = (x: number, y: number, z: number, w: number) => { + return packUnorm(x, 8) << 24 | + packUnorm(y, 8) << 16 | + packUnorm(z, 8) << 8 | + packUnorm(w, 8); + }; + + // pack quaternion into 2,10,10,10 + const packRot = (x: number, y: number, z: number, w: number) => { + q.set(x, y, z, w).normalize(); + const a = [q.x, q.y, q.z, q.w]; + const largest = a.reduce((curr, v, i) => Math.abs(v) > Math.abs(a[curr]) ? i : curr, 0); + + if (a[largest] < 0) { + a[0] = -a[0]; + a[1] = -a[1]; + a[2] = -a[2]; + a[3] = -a[3]; + } + + const norm = Math.sqrt(2) * 0.5; + let result = largest; + for (let i = 0; i < 4; ++i) { + if (i !== largest) { + result = (result << 10) | packUnorm(a[i] * norm + 0.5, 10); + } + } + + return result; + }; + + const packColor = (r: number, g: number, b: number, a: number) => { + const SH_C0 = 0.28209479177387814; + return pack8888( + r * SH_C0 + 0.5, + g * SH_C0 + 0.5, + b * SH_C0 + 0.5, + 1 / (1 + Math.exp(-a)) + ); + }; + + // pack + for (let i = 0; i < this.size; ++i) { + this.position[i] = pack111011( + (x[i] - px.min) / (px.max - px.min), + (y[i] - py.min) / (py.max - py.min), + (z[i] - pz.min) / (pz.max - pz.min) + ); + + this.rotation[i] = packRot(rot_0[i], rot_1[i], rot_2[i], rot_3[i]); + + this.scale[i] = pack111011( + (scale_0[i] - sx.min) / (sx.max - sx.min), + (scale_1[i] - sy.min) / (sy.max - sy.min), + (scale_2[i] - sz.min) / (sz.max - sz.min) + ); + + this.color[i] = packColor(f_dc_0[i], f_dc_1[i], f_dc_2[i], opacity[i]); + } + + return { px, py, pz, sx, sy, sz }; + } +} + +const convertPlyCompressed = (splatData: SplatData, modelMat: Mat4) => { + const sortSplats = (indices: number[]) => { + // https://fgiesen.wordpress.com/2009/12/13/decoding-morton-codes/ + const encodeMorton3 = (x: number, y: number, z: number) : number => { + const Part1By2 = (x: number) => { + x &= 0x000003ff; + x = (x ^ (x << 16)) & 0xff0000ff; + x = (x ^ (x << 8)) & 0x0300f00f; + x = (x ^ (x << 4)) & 0x030c30c3; + x = (x ^ (x << 2)) & 0x09249249; + return x; + }; + + return (Part1By2(z) << 2) + (Part1By2(y) << 1) + Part1By2(x); + }; + + const x = splatData.getProp('x'); + const y = splatData.getProp('y'); + const z = splatData.getProp('z'); + + const bx = calcMinMax(x, indices); + const by = calcMinMax(y, indices); + const bz = calcMinMax(z, indices); + + // generate morton codes + const morton = indices.map((i) => { + const ix = Math.floor(1024 * (x[i] - bx.min) / (bx.max - bx.min)); + const iy = Math.floor(1024 * (y[i] - by.min) / (by.max - by.min)); + const iz = Math.floor(1024 * (z[i] - bz.min) / (bz.max - bz.min)); + return encodeMorton3(ix, iy, iz); + }); + + // order splats by morton code + indices.sort((a, b) => morton[a] - morton[b]); + }; + + // generate index list of surviving splats + const opacity = splatData.getProp('opacity'); + const indices = []; + for (let i = 0; i < splatData.numSplats; ++i) { + if (opacity[i] !== deletedOpacity) { + indices.push(i); + } + } + + if (indices.length === 0) { + console.error('nothing to export'); + return; + } + + const numSplats = indices.length; + const numChunks = Math.ceil(numSplats / 256); + + const chunkProps = ['min_x', 'min_y', 'min_z', 'max_x', 'max_y', 'max_z', 'min_scale_x', 'min_scale_y', 'min_scale_z', 'max_scale_x', 'max_scale_y', 'max_scale_z']; + const vertexProps = ['packed_position', 'packed_rotation', 'packed_scale', 'packed_color']; + const headerText = [ + [ + `ply`, + `format binary_little_endian 1.0`, + `comment generated by super-splat`, + `element chunk ${numChunks}` + ], + chunkProps.map(p => `property float ${p}`), + [ + `element vertex ${numSplats}` + ], + vertexProps.map(p => `property uint ${p}`), + [ + `end_header\n` + ] + ].flat().join('\n'); + + const header = (new TextEncoder()).encode(headerText); + const result = new Uint8Array(header.byteLength + numChunks * chunkProps.length * 4 + numSplats * vertexProps.length * 4); + const dataView = new DataView(result.buffer); + + result.set(header); + + const chunkOffset = header.byteLength; + const vertexOffset = chunkOffset + numChunks * 12 * 4; + + const chunk = new Chunk(); + + // sort splats into some kind of order + sortSplats(indices); + + for (let i = 0; i < numChunks; ++i) { + chunk.set(splatData, indices.slice(i * 256, (i + 1) * 256)); + chunk.transform(modelMat); + + const result = chunk.pack(); + + // write chunk data + dataView.setFloat32(chunkOffset + i * 12 * 4 + 0, result.px.min, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 4, result.py.min, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 8, result.pz.min, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 12, result.px.max, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 16, result.py.max, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 20, result.pz.max, true); + + dataView.setFloat32(chunkOffset + i * 12 * 4 + 24, result.sx.min, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 28, result.sy.min, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 32, result.sz.min, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 36, result.sx.max, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 40, result.sy.max, true); + dataView.setFloat32(chunkOffset + i * 12 * 4 + 44, result.sz.max, true); + + // write splat data + let offset = vertexOffset + i * 256 * 4 * 4; + const chunkSplats = Math.min(numSplats, (i + 1) * 256) - i * 256; + for (let j = 0; j < chunkSplats; ++j) { + dataView.setUint32(offset + j * 4 * 4 + 0, chunk.position[j], true); + dataView.setUint32(offset + j * 4 * 4 + 4, chunk.rotation[j], true); + dataView.setUint32(offset + j * 4 * 4 + 8, chunk.scale[j], true); + dataView.setUint32(offset + j * 4 * 4 + 12, chunk.color[j], true); + } + } + + return result; +}; + +const convertSplat = (splatData: SplatData, modelMat: Mat4) => { + // count the number of non-deleted splats + const x = splatData.getProp('x'); + const y = splatData.getProp('y'); + const z = splatData.getProp('z'); + const opacity = splatData.getProp('opacity'); + const rot_0 = splatData.getProp('rot_0'); + const rot_1 = splatData.getProp('rot_1'); + const rot_2 = splatData.getProp('rot_2'); + const rot_3 = splatData.getProp('rot_3'); + const f_dc_0 = splatData.getProp('f_dc_0'); + const f_dc_1 = splatData.getProp('f_dc_1'); + const f_dc_2 = splatData.getProp('f_dc_2'); + const scale_0 = splatData.getProp('scale_0'); + const scale_1 = splatData.getProp('scale_1'); + const scale_2 = splatData.getProp('scale_2'); + + // count number of non-deleted splats + let numSplats = 0; + for (let i = 0; i < splatData.numSplats; ++i) { + numSplats += opacity[i] !== deletedOpacity ? 1 : 0; + } + + // position.xyz: float32, scale.xyz: float32, color.rgba: uint8, quaternion.ijkl: uint8 + const result = new Uint8Array(numSplats * 32); + const dataView = new DataView(result.buffer); + + // we must undo the transform we apply at load time to output data + const mat = new Mat4(); + mat.setScale(-1, -1, 1); + mat.invert(); + mat.mul2(mat, modelMat); + + const quat = new Quat(); + quat.setFromMat4(mat); + + const v = new Vec3(); + const q = new Quat(); + + const scale = new Vec3(); + mat.getScale(scale); + + const clamp = (x: number) => Math.max(0, Math.min(255, x)); + let idx = 0; + + for (let i = 0; i < splatData.numSplats; ++i) { + if (opacity[i] === deletedOpacity) continue; + + const off = idx++ * 32; + + v.set(x[i], y[i], z[i]); + mat.transformPoint(v, v); + dataView.setFloat32(off + 0, v.x, true); + dataView.setFloat32(off + 4, v.y, true); + dataView.setFloat32(off + 8, v.z, true); + + dataView.setFloat32(off + 12, Math.exp(scale_0[i]) * scale.x, true); + dataView.setFloat32(off + 16, Math.exp(scale_1[i]) * scale.x, true); + dataView.setFloat32(off + 20, Math.exp(scale_2[i]) * scale.x, true); + + const SH_C0 = 0.28209479177387814; + dataView.setUint8(off + 24, clamp((0.5 + SH_C0 * f_dc_0[i]) * 255)); + dataView.setUint8(off + 25, clamp((0.5 + SH_C0 * f_dc_1[i]) * 255)); + dataView.setUint8(off + 26, clamp((0.5 + SH_C0 * f_dc_2[i]) * 255)); + dataView.setUint8(off + 27, clamp((1 / (1 + Math.exp(-opacity[i]))) * 255)); + + q.set(rot_1[i], rot_2[i], rot_3[i], rot_0[i]).mul2(quat, q).normalize(); + dataView.setUint8(off + 28, clamp(q.w * 128 + 128)); + dataView.setUint8(off + 29, clamp(q.x * 128 + 128)); + dataView.setUint8(off + 30, clamp(q.y * 128 + 128)); + dataView.setUint8(off + 31, clamp(q.z * 128 + 128)); + } + + return result; +}; + +export { convertPly, convertPlyCompressed, convertSplat }; diff --git a/src/splat-debug.ts b/src/splat-debug.ts new file mode 100644 index 0000000..8d7a54f --- /dev/null +++ b/src/splat-debug.ts @@ -0,0 +1,117 @@ +import { + BLEND_NORMAL, + Material, + Mesh, + MeshInstance, + PRIMITIVE_POINTS, + SEMANTIC_POSITION, + createShaderFromCode, +} from 'playcanvas'; +import { deletedOpacity } from './edit-ops'; +import { Scene } from './scene'; +import { Splat } from './splat'; +import { SplatData } from 'playcanvas-extras'; + +const vs = /* glsl */ ` +attribute vec4 vertex_position; + +uniform mat4 matrix_model; +uniform mat4 matrix_view; +uniform mat4 matrix_projection; +uniform mat4 matrix_viewProjection; + +uniform float splatSize; + +varying vec4 color; + +void main(void) { + if (vertex_position.w == -1.0) { + gl_Position = vec4(0.0, 0.0, 2.0, 1.0); + } else { + gl_Position = matrix_viewProjection * matrix_model * vec4(vertex_position.xyz, 1.0); + gl_PointSize = splatSize; + float opacity = vertex_position.w; + color = (opacity == -1.0) ? vec4(0) : mix(vec4(0, 0, 1.0, 0.5), vec4(1.0, 1.0, 0.0, 0.5), opacity); + } +} +`; + +const fs = /* glsl */ ` +varying vec4 color; +void main(void) +{ + gl_FragColor = color; +} +`; + +class SplatDebug { + splatData: SplatData; + meshInstance: MeshInstance; + + constructor(scene: Scene, splat: Splat, splatData: SplatData) { + const device = scene.graphicsDevice; + + const shader = createShaderFromCode(device, vs, fs, `splatDebugShader`, { + vertex_position: SEMANTIC_POSITION + }); + + const material = new Material(); + material.name = 'splatDebugMaterial'; + material.blendType = BLEND_NORMAL; + material.shader = shader; + material.setParameter('splatSize', 1.0); + material.update(); + + const x = splatData.getProp('x'); + const y = splatData.getProp('y'); + const z = splatData.getProp('z'); + const s = splatData.getProp('selection'); + + const vertexData = new Float32Array(splatData.numSplats * 4); + for (let i = 0; i < splatData.numSplats; ++i) { + vertexData[i * 4 + 0] = x[i]; + vertexData[i * 4 + 1] = y[i]; + vertexData[i * 4 + 2] = z[i]; + vertexData[i * 4 + 3] = s[i]; + } + + const mesh = new Mesh(device); + mesh.setPositions(vertexData, 4); + mesh.update(PRIMITIVE_POINTS, true); + + this.splatData = splatData; + this.meshInstance = new MeshInstance(mesh, material, splat.root); + } + + update() { + const splatData = this.splatData; + const s = splatData.getProp('selection'); + const o = splatData.getProp('opacity'); + + const vb = this.meshInstance.mesh.vertexBuffer; + const vertexData = new Float32Array(vb.lock()); + + let count = 0; + + for (let i = 0; i < splatData.numSplats; ++i) { + const selection = o[i] === deletedOpacity ? -1 : s[i]; + vertexData[i * 4 + 3] = selection; + count += selection === 1 ? 1 : 0; + } + + vb.unlock(); + + return count; + } + + set splatSize(splatSize: number) { + this.meshInstance.material.setParameter('splatSize', splatSize); + } + + get splatSize() { + // @ts-ignore + return this.meshInstance.material.getParameter('splatSize').data; + } +} + +export { SplatDebug }; diff --git a/src/style.scss b/src/style.scss index 9307402..5a00702 100644 --- a/src/style.scss +++ b/src/style.scss @@ -162,6 +162,27 @@ body { margin-right: 6px; } +.error-popup { + position: absolute; + left: 0; + right: 0; + margin-left: auto; + margin-right: auto; + width: 400px; + top: 50%; + color: red; +} + +.info-popup { + position: absolute; + left: 0; + right: 0; + margin-left: auto; + margin-right: auto; + width: 400px; + top: 50%; +} + /* scrollbar styling */ ::-webkit-scrollbar { width: 8px; diff --git a/src/ui/control-panel.ts b/src/ui/control-panel.ts index 4f0a567..88b5e4d 100644 --- a/src/ui/control-panel.ts +++ b/src/ui/control-panel.ts @@ -659,12 +659,18 @@ class ControlPanel extends Container { text: 'Ply file' }); + const exportCompressedPlyButton = new Button({ + class: 'control-element', + text: 'Compressed Ply file' + }); + const exportSplatButton = new Button({ class: 'control-element', text: 'Splat file' }); exportPanel.append(exportPlyButton); + exportPanel.append(exportCompressedPlyButton); exportPanel.append(exportSplatButton); // keyboard @@ -867,6 +873,10 @@ class ControlPanel extends Container { this.events.fire('export', 'ply'); }); + exportCompressedPlyButton.on('click', () => { + this.events.fire('export', 'ply-compressed'); + }); + exportSplatButton.on('click', () => { this.events.fire('export', 'splat'); }); diff --git a/src/editor-ui.ts b/src/ui/editor.ts similarity index 64% rename from src/editor-ui.ts rename to src/ui/editor.ts index b1e3f02..ed2382b 100644 --- a/src/editor-ui.ts +++ b/src/ui/editor.ts @@ -1,7 +1,7 @@ -import { Container } from 'pcui'; -import { version as supersplatVersion } from '../package.json'; -import logo from './ui/playcanvas-logo.png'; -import { ControlPanel } from './ui/control-panel'; +import { Container, InfoBox } from 'pcui'; +import { version as supersplatVersion } from '../../package.json'; +import { ControlPanel } from './control-panel'; +import logo from './playcanvas-logo.png'; class EditorUI { appContainer: Container; @@ -9,6 +9,8 @@ class EditorUI { controlPanel: ControlPanel; canvasContainer: Container; canvas: HTMLCanvasElement; + errorPopup: InfoBox; + infoPopup: InfoBox; constructor() { // favicon @@ -35,9 +37,28 @@ class EditorUI { const canvas = document.createElement('canvas'); canvas.id = 'canvas'; + canvasContainer.dom.appendChild(canvas); + + // error box + const errorPopup = new InfoBox({ + class: 'error-popup', + icon: 'E218', + title: 'Error', + hidden: true + }); + + // info box + const infoPopup = new InfoBox({ + class: 'info-popup', + icon: 'E400', + title: 'Info', + hidden: true + }); + appContainer.append(leftContainer); appContainer.append(canvasContainer); - canvasContainer.dom.appendChild(canvas); + appContainer.append(errorPopup); + appContainer.append(infoPopup); // title const title = new Container({ @@ -76,6 +97,26 @@ class EditorUI { this.controlPanel = controlPanel; this.canvasContainer = canvasContainer; this.canvas = canvas; + this.errorPopup = errorPopup; + this.infoPopup = infoPopup; + } + + showError(err: string) { + if (err) { + this.errorPopup.text = err; + this.errorPopup.hidden = false; + } else { + this.errorPopup.hidden = true; + } + } + + showInfo(info: string) { + if (info) { + this.infoPopup.text = info; + this.infoPopup.hidden = false; + } else { + this.infoPopup.hidden = true; + } } } diff --git a/src/spinner.ts b/src/ui/spinner.ts similarity index 100% rename from src/spinner.ts rename to src/ui/spinner.ts diff --git a/submodules/engine b/submodules/engine index e074b37..38a4550 160000 --- a/submodules/engine +++ b/submodules/engine @@ -1 +1 @@ -Subproject commit e074b37a9ab7ff3dd00aa6e639c75408775a7521 +Subproject commit 38a455076a4c88776a33196026fba88fac7b7c87 diff --git a/tsconfig.json b/tsconfig.json index e24a77f..b5532c5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,6 @@ "pcui": ["node_modules/@playcanvas/pcui"] } }, - "include": ["./src/*.ts","./submodules/engine/extras/*/*.ts", "src/ui/control-panel.ts"], + "include": ["./src/*.ts", "src/ui/*.ts", "./submodules/engine/extras/*/*.ts"], "exclude": ["node_modules", "**/*.js", "dist"] }