Fix point cloud jitter and quantization issues for additive refinement pnts

common/api/core-frontend.api.md

@@ -8706,7 +8706,10 @@ export class ReadonlyTileUserSet extends ReadonlySortedArray<TileUser> {
 }
 
 // @internal
-export function readPointCloudTileContent(stream: ByteStream, iModel: IModelConnection, modelId: Id64String, _is3d: boolean, range: ElementAlignedBox3d, system: RenderSystem): Promise<RenderGraphic | undefined>;
+export function readPointCloudTileContent(stream: ByteStream, iModel: IModelConnection, modelId: Id64String, _is3d: boolean, range: ElementAlignedBox3d, system: RenderSystem): Promise<{
+    graphic: RenderGraphic | undefined;
+    rtcCenter: Point3d | undefined;
+}>;
 
 // @alpha
 export class RealityDataError extends BentleyError {

common/api/summary/core-frontend.exports.csv

@@ -535,7 +535,6 @@ public;ReadGltfGraphicsArgs
 public;ReadImageBufferArgs
 public;ReadMeshArgs
 internal;ReadonlyTileUserSet 
-internal;readPointCloudTileContent(stream: ByteStream, iModel: IModelConnection, modelId: Id64String, _is3d: boolean, range: ElementAlignedBox3d, system: RenderSystem): Promise
 alpha;RealityDataError 
 beta;RealityDataSource
 beta;RealityDataSource

common/changes/@itwin/core-frontend/djs-fix-point-jitter_2023-04-04-00-27.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@itwin/core-frontend",
+      "comment": "added return of rtcCenter to readPointCloudTileContent",
+      "type": "none"
+    }
+  ],
+  "packageName": "@itwin/core-frontend"
+}

core/frontend/src/render/primitives/PointCloudPrimitive.ts

@@ -9,7 +9,7 @@
 import { FeatureIndex, QParams3d } from "@itwin/core-common";
 
 export interface PointCloudArgs {
-  positions: Uint8Array | Uint16Array;
+  positions: Uint8Array | Uint16Array | Float32Array;
   qparams: QParams3d;
   colors: Uint8Array;
   features: FeatureIndex;

core/frontend/src/render/webgl/AttributeBuffers.ts

@@ -337,7 +337,7 @@ export class QBufferHandle3d extends BufferHandle {
     this.scale = qscale3dToArray(qParams.scale);
   }
 
-  public static create(qParams: QParams3d, data: Uint16Array | Uint8Array): QBufferHandle3d | undefined {
+  public static create(qParams: QParams3d, data: Uint16Array | Uint8Array | Float32Array): QBufferHandle3d | undefined {
     const handle = new QBufferHandle3d(qParams);
     if (handle.isDisposed) {
       return undefined;

core/frontend/src/render/webgl/PointCloud.ts

@@ -48,7 +48,7 @@ export class PointCloudGeometry extends CachedGeometry {
     this._vertices = QBufferHandle3d.create(pointCloud.qparams, pointCloud.positions) as QBufferHandle3d;
     const attrPos = AttributeMap.findAttribute("a_pos", TechniqueId.PointCloud, false);
     assert(undefined !== attrPos);
-    const vertexDataType = (pointCloud.positions instanceof Uint8Array) ? GL.DataType.UnsignedByte : GL.DataType.UnsignedShort;
+    const vertexDataType = (pointCloud.positions instanceof Float32Array) ? GL.DataType.Float : ((pointCloud.positions instanceof Uint8Array) ? GL.DataType.UnsignedByte : GL.DataType.UnsignedShort);
     this.buffers.addBuffer(this._vertices, [BufferParameters.create(attrPos.location, 3, vertexDataType, false, 0, 0, false)]);
     this._vertexCount = pointCloud.positions.length / 3;
     this._hasFeatures = FeatureIndexType.Empty !== pointCloud.features.type;

core/frontend/src/tile/PntsReader.ts

@@ -7,7 +7,7 @@
  */
 
 import { ByteStream, Id64String, Logger, utf8ToString } from "@itwin/core-bentley";
-import { Point3d, Range3d, Vector3d } from "@itwin/core-geometry";
+import { Point3d, Range3d } from "@itwin/core-geometry";
 import { BatchType, ElementAlignedBox3d, Feature, FeatureTable, PackedFeatureTable, PntsHeader, QParams3d, QPoint3d, Quantization } from "@itwin/core-common";
 import { FrontendLoggerCategory } from "../FrontendLoggerCategory";
 import { IModelConnection } from "../IModelConnection";
@@ -33,7 +33,7 @@ interface DracoPointCloud {
 
 interface PointCloudProps {
   params: QParams3d;
-  points: Uint16Array;
+  points: Uint16Array | Float32Array;
   colors?: Uint8Array;
 }
 
@@ -118,7 +118,7 @@ function readPntsColors(stream: ByteStream, dataOffset: number, pnts: PntsProps)
 function readPnts(stream: ByteStream, dataOffset: number, pnts: PntsProps): PointCloudProps | undefined {
   const nPts = pnts.POINTS_LENGTH;
   let params: QParams3d;
-  let points: Uint16Array;
+  let points: Uint16Array | Float32Array;
 
   if (pnts.POSITION_QUANTIZED) {
     const qpos = pnts.POSITION_QUANTIZED;
@@ -131,23 +131,10 @@ function readPnts(stream: ByteStream, dataOffset: number, pnts: PntsProps): Poin
     params = QParams3d.fromOriginAndScale(qOrigin, qScale);
     points = new Uint16Array(stream.arrayBuffer, dataOffset + qpos.byteOffset, 3 * nPts);
   } else {
-    const nCoords = nPts * 3;
-    const fpts = new Float32Array(stream.arrayBuffer, dataOffset + pnts.POSITION.byteOffset, 3 * nPts);
-    const range = Range3d.createNull();
-    for (let i = 0; i < nCoords; i += 3)
-      range.extendXYZ(fpts[i], fpts[i + 1], fpts[i + 2]);
-
-    params = QParams3d.fromRange(range);
-    const qpt = new QPoint3d();
-    const fpt = new Point3d();
-    points = new Uint16Array(3 * nPts);
-    for (let i = 0; i < nCoords; i += 3) {
-      fpt.set(fpts[i], fpts[i + 1], fpts[i + 2]);
-      qpt.init(fpt, params);
-      points[i] = qpt.x;
-      points[i + 1] = qpt.y;
-      points[i + 2] = qpt.z;
-    }
+    const qOrigin = new Point3d(0, 0, 0);
+    const qScale = new Point3d(1, 1, 1);
+    params = QParams3d.fromOriginAndScale(qOrigin, qScale);
+    points = new Float32Array(stream.arrayBuffer, dataOffset + pnts.POSITION.byteOffset, 3 * nPts);
   }
 
   const colors = readPntsColors(stream, dataOffset, pnts);
@@ -157,7 +144,7 @@ function readPnts(stream: ByteStream, dataOffset: number, pnts: PntsProps): Poin
 async function decodeDracoPointCloud(buf: Uint8Array): Promise<PointCloudProps | undefined> {
   try {
     const dracoLoader = (await import("@loaders.gl/draco")).DracoLoader;
-    const mesh = await dracoLoader.parse(buf, { });
+    const mesh = await dracoLoader.parse(buf, {});
     if (mesh.topology !== "point-list")
       return undefined;
 
@@ -214,18 +201,20 @@ async function decodeDracoPointCloud(buf: Uint8Array): Promise<PointCloudProps |
 /** Deserialize a point cloud tile and return it as a RenderGraphic.
  * @internal
  */
-export async function readPointCloudTileContent(stream: ByteStream, iModel: IModelConnection, modelId: Id64String, _is3d: boolean, range: ElementAlignedBox3d, system: RenderSystem): Promise<RenderGraphic | undefined> {
+export async function readPointCloudTileContent(stream: ByteStream, iModel: IModelConnection, modelId: Id64String, _is3d: boolean, range: ElementAlignedBox3d, system: RenderSystem): Promise<{ graphic: RenderGraphic | undefined, rtcCenter: Point3d | undefined }> {
+  let graphic;
+  let rtcCenter;
   const header = new PntsHeader(stream);
   if (!header.isValid)
-    return undefined;
+    return { graphic, rtcCenter };
 
   const featureTableJsonOffset = stream.curPos;
   const featureStrData = stream.nextBytes(header.featureTableJsonLength);
   const featureStr = utf8ToString(featureStrData);
   const featureValue = JSON.parse(featureStr as string) as PntsProps;
 
   if (undefined === featureValue)
-    return undefined;
+    return { graphic, rtcCenter };
 
   let props: PointCloudProps | undefined;
   const dataOffset = featureTableJsonOffset + header.featureTableJsonLength;
@@ -242,10 +231,15 @@ export async function readPointCloudTileContent(stream: ByteStream, iModel: IMod
   }
 
   if (!props)
-    return undefined;
-
-  if (featureValue.RTC_CENTER)
-    props.params = QParams3d.fromOriginAndScale(props.params.origin.plus(Vector3d.fromJSON(featureValue.RTC_CENTER)), props.params.scale);
+    return { graphic, rtcCenter };
+
+  let batchRange = range;
+  if (featureValue.RTC_CENTER) {
+    rtcCenter = Point3d.fromJSON(featureValue.RTC_CENTER);
+    batchRange = range.clone();
+    batchRange.low.minus(rtcCenter, batchRange.low);
+    batchRange.high.minus(rtcCenter, batchRange.high);
+  }
 
   if (!props.colors) {
     // ###TODO we really should support uniform color instead of allocating an RGB value per point...
@@ -267,9 +261,17 @@ export async function readPointCloudTileContent(stream: ByteStream, iModel: IMod
   const featureTable = new FeatureTable(1, modelId, BatchType.Primary);
   const features = new Mesh.Features(featureTable);
   features.add(new Feature(modelId), 1);
-  const voxelSize = props.params.rangeDiagonal.maxAbs() / 256;
+  let params = props.params;
+  if (props.points instanceof Float32Array) {
+    // we don't have a true range for unquantized points, so calc one here for voxelSize
+    const rng = Range3d.createNull();
+    for (let i = 0; i < props.points.length; i += 3)
+      rng.extendXYZ(props.points[i], props.points[i + 1], props.points[i + 2]);
+    params = QParams3d.fromRange(rng);
+  }
+  const voxelSize = params.rangeDiagonal.maxAbs() / 256;
 
-  let renderGraphic = system.createPointCloud({
+  graphic = system.createPointCloud({
     positions: props.points,
     qparams: props.params,
     colors: props.colors,
@@ -278,6 +280,6 @@ export async function readPointCloudTileContent(stream: ByteStream, iModel: IMod
     colorFormat: "rgb",
   }, iModel);
 
-  renderGraphic = system.createBatch(renderGraphic!, PackedFeatureTable.pack(featureTable), range);
-  return renderGraphic;
+  graphic = system.createBatch(graphic!, PackedFeatureTable.pack(featureTable), batchRange);
+  return { graphic, rtcCenter };
 }

core/frontend/src/tile/RealityTileLoader.ts

@@ -77,7 +77,7 @@ export abstract class RealityTileLoader {
 
   }
 
-  public async loadGeometryFromStream(tile: RealityTile,  streamBuffer: ByteStream, system: RenderSystem): Promise<RealityTileContent> {
+  public async loadGeometryFromStream(tile: RealityTile, streamBuffer: ByteStream, system: RenderSystem): Promise<RealityTileContent> {
     const format = this._getFormat(streamBuffer);
     if (format !== TileFormat.B3dm)
       return {};
@@ -110,11 +110,22 @@ export abstract class RealityTileLoader {
         break;
       case TileFormat.Pnts:
         this._containsPointClouds = true;
-        let graphic = await readPointCloudTileContent(streamBuffer, iModel, modelId, is3d, tile.contentRange, system);
-        if (graphic && tile.transformToRoot && !tile.transformToRoot.isIdentity) {
+        const res = await readPointCloudTileContent(streamBuffer, iModel, modelId, is3d, tile.contentRange, system);
+        let graphic = res.graphic;
+        const rtcCenter = res.rtcCenter;
+        if (graphic && (rtcCenter || tile.transformToRoot && !tile.transformToRoot.isIdentity)) {
           const transformBranch = new GraphicBranch(true);
           transformBranch.add(graphic);
-          graphic = system.createBranch(transformBranch, tile.transformToRoot);
+          let xform: Transform;
+          if (!tile.transformToRoot && rtcCenter)
+            xform = Transform.createTranslation(rtcCenter);
+          else {
+            if (rtcCenter)
+              xform = Transform.createOriginAndMatrix(rtcCenter.plus(tile.transformToRoot!.origin), tile.transformToRoot!.matrix);
+            else
+              xform = tile.transformToRoot!;
+          }
+          graphic = system.createBranch(transformBranch, xform);
         }
 
         return { graphic };
@@ -199,7 +210,7 @@ export abstract class RealityTileLoader {
 
       if (currentInputState.viewport === viewport && viewport instanceof ScreenViewport) {
         // Try to get a better target point from the last zoom target
-        const {lastWheelEvent} = currentInputState;
+        const { lastWheelEvent } = currentInputState;
 
         if (lastWheelEvent !== undefined && now - lastWheelEvent.time < wheelEventRelevanceTimeout) {
           const focusPointCandidate = Point2d.fromJSON(viewport.worldToNpc(lastWheelEvent.point));