Merge pull request #4120 from voxel51/merge/v0.23.6-to-main

Merge `release/v0.23.6` to `main`

app/packages/core/src/components/Filters/StringFilter/state.ts

@@ -40,7 +40,7 @@ export const pathSearchCount = selectorFamily({
             path,
             filter: { path, value },
           })
-        )?.values?.[0].count || 0
+        )?.values?.[0]?.count || 0
       );
     },
 });

app/packages/core/src/useFlashlightPager.ts

@@ -2,6 +2,7 @@ import { FlashlightConfig, Response } from "@fiftyone/flashlight";
 import { zoomAspectRatio } from "@fiftyone/looker";
 import * as foq from "@fiftyone/relay";
 import * as fos from "@fiftyone/state";
+import { Schema } from "@fiftyone/utilities";
 import { useMemo, useRef, useState } from "react";
 import { useErrorHandler } from "react-error-boundary";
 import { VariablesOf, fetchQuery, useRelayEnvironment } from "react-relay";
@@ -13,6 +14,7 @@ const processSamplePageData = (
   offset: number,
   store: fos.LookerStore<fos.Lookers>,
   data: fos.ResponseFrom<foq.paginateSamplesQuery>,
+  schema: Schema,
   zoom?: boolean
 ) => {
   return data.samples.edges.map((edge, i) => {
@@ -26,7 +28,7 @@ const processSamplePageData = (
 
     return {
       aspectRatio: zoom
-        ? zoomAspectRatio(edge.node.sample, edge.node.aspectRatio)
+        ? zoomAspectRatio(edge.node.sample, schema, edge.node.aspectRatio)
         : edge.node.aspectRatio,
       id: edge.node.id,
     };
@@ -50,6 +52,9 @@ const useFlashlightPager = (
   const zoom = useRecoilValue(zoomSelector || defaultZoom);
   const [isEmpty, setIsEmpty] = useState(false);
   const handleError = useErrorHandler();
+  const schema = useRecoilValue(
+    fos.fieldSchema({ space: fos.State.SPACE.SAMPLE })
+  );
 
   const pager = useMemo(() => {
     return async (pageNumber: number) => {
@@ -66,6 +71,7 @@ const useFlashlightPager = (
               pageNumber * PAGE_SIZE,
               store,
               data,
+              schema,
               zoomValue
             );
 
@@ -83,7 +89,7 @@ const useFlashlightPager = (
         });
       });
     };
-  }, [environment, handleError, page, store, zoom]);
+  }, [environment, handleError, page, schema, store, zoom]);
 
   const ref = useRef<FlashlightConfig<number>["get"]>(pager);
   ref.current = pager;

app/packages/looker-3d/src/Looker3d.tsx

@@ -1,13 +1,11 @@
 import { Loading, useTheme } from "@fiftyone/components";
 import {
-  getHashLabel,
   getLabelColor,
-  isValidColor,
   shouldShowLabelTag,
 } from "@fiftyone/looker/src/overlays/util";
 import * as fop from "@fiftyone/plugins";
 import * as fos from "@fiftyone/state";
-import { COLOR_BY, getColor } from "@fiftyone/utilities";
+import { fieldSchema } from "@fiftyone/state";
 import { Typography } from "@mui/material";
 import { OrbitControlsProps as OrbitControls } from "@react-three/drei";
 import { Canvas } from "@react-three/fiber";
@@ -108,7 +106,7 @@ export const Looker3d = () => {
       onSelectLabel({
         detail: {
           id: label._id,
-          field: label.path[label.path.length - 1],
+          field: label.path,
           sampleId: label.sampleId,
         },
       });
@@ -397,25 +395,26 @@ export const Looker3d = () => {
       isPointcloudDataset,
     ]
   );
+  const schema = useRecoilValue(fieldSchema({ space: fos.State.SPACE.SAMPLE }));
 
   const overlays = useMemo(
     () =>
-      load3dOverlays(sampleMap, selectedLabels)
+      load3dOverlays(sampleMap, selectedLabels, [], schema)
         .map((l) => {
-          const path = l.path.join(".");
           const isTagged = shouldShowLabelTag(selectedLabelTags, l.tags);
           const color = getLabelColor({
             coloring,
-            path,
+            path: l.path,
             label: l,
             isTagged,
             labelTagColors,
             customizeColorSetting,
+            embeddedDocType: l._cls,
           });
 
           return { ...l, color, id: l._id };
         })
-        .filter((l) => pathFilter(l.path.join("."), l)),
+        .filter((l) => pathFilter(l.path, l)),
     [
       coloring,
       getFieldColor,
@@ -424,6 +423,7 @@ export const Looker3d = () => {
       selectedLabels,
       colorSchemeFields,
       colorScheme,
+      schema,
     ]
   );
 

app/packages/looker-3d/src/overlays/loader.ts

@@ -1,24 +1,27 @@
 import * as fos from "@fiftyone/looker/src/state";
 import { SampleData } from "@fiftyone/state";
+import { getCls, LABEL_LIST, Schema } from "@fiftyone/utilities";
 
 const RENDERABLE = ["Detection", "Polyline"];
 const RENDERABLE_LIST = ["Detections", "Polylines"];
 
 export type OverlayLabel = {
   _id: string;
-  _cls: string;
-  path: string[];
+  path: string;
   selected: boolean;
   color?: string;
   label?: string;
   sampleId?: string;
+  _cls: string;
 };
 
 export const load3dOverlayForSample = (
   sampleId: string,
   samples: SampleData | fos.Sample[],
   selectedLabels: Record<string, unknown>,
-  currentPath = []
+  currentPath: string[] = [],
+  schema: Schema,
+  rest: string[] = []
 ) => {
   let overlays: OverlayLabel[] = [];
 
@@ -27,26 +30,34 @@ export const load3dOverlayForSample = (
 
   for (let i = 0; i < labelValues.length; i++) {
     const label = labelValues[i];
+
     const labelKey = labelKeys ? labelKeys[i] : "";
+
     if (!label) {
       continue;
     }
 
-    if (RENDERABLE.includes(label._cls)) {
+    const path = [...currentPath, labelKey].filter((k) => !!k).join(".");
+    const cls = getCls([path, ...rest].join("."), schema);
+
+    if (RENDERABLE.includes(cls)) {
       overlays.push({
         ...label,
         sampleId,
-        path: [...currentPath, labelKey].filter((k) => !!k),
+        path,
         selected: label._id in selectedLabels,
+        _cls: cls,
       });
-    } else if (RENDERABLE_LIST.includes(label._cls)) {
+    } else if (RENDERABLE_LIST.includes(cls) && label[LABEL_LIST[cls]]) {
       overlays = [
         ...overlays,
         ...load3dOverlayForSample(
           sampleId,
-          label[label._cls.toLowerCase()],
+          label[LABEL_LIST[cls]],
           selectedLabels,
-          labelKey ? [...currentPath, labelKey] : [...currentPath]
+          [...currentPath, labelKey],
+          schema,
+          [LABEL_LIST[cls]]
         ),
       ];
     }
@@ -58,7 +69,8 @@ export const load3dOverlayForSample = (
 export const load3dOverlays = (
   samples: { [sliceOrFilename: string]: SampleData } | fos.Sample[],
   selectedLabels: Record<string, unknown>,
-  currentPath = []
+  currentPath: string[] = [],
+  schema: Schema
 ) => {
   const overlays = [];
   for (const [_sliceOrFilename, sampleWrapper] of Object.entries(samples)) {
@@ -71,7 +83,8 @@ export const load3dOverlays = (
         sampleWrapper.sample._id,
         sampleWrapper.sample,
         selectedLabels,
-        currentPath
+        currentPath,
+        schema
       )
     );
   }

app/packages/looker/src/elements/common/util.test.ts

@@ -36,7 +36,6 @@ describe("EmbeddedDocumentField and tags bubbles get correct color", () => {
     id: "1",
     label: "telephone",
     tags: ["non-worm", "mistake"],
-    _cls: "Classification",
   };
 
   const sampleInput1 = {

app/packages/looker/src/lookers/abstract.ts

@@ -181,7 +181,7 @@ export abstract class AbstractLooker<
   }
 
   loadOverlays(sample: Sample): void {
-    this.sampleOverlays = loadOverlays(sample);
+    this.sampleOverlays = loadOverlays(sample, this.state.config.fieldSchema);
   }
 
   pluckOverlays(state: Readonly<State>): Overlay<State>[] {
@@ -700,6 +700,7 @@ export abstract class AbstractLooker<
       labelTagColors: this.state.options.labelTagColors,
       selectedLabelTags: this.state.options.selectedLabelTags,
       sources: this.state.config.sources,
+      schema: this.state.config.fieldSchema,
       uuid: messageUUID,
     } as ProcessSample);
   }

app/packages/looker/src/lookers/video.ts

@@ -14,13 +14,13 @@ import {
   FrameChunkResponse,
   FrameSample,
   LabelData,
-  Optional,
   StateUpdate,
   VideoSample,
   VideoState,
 } from "../state";
 import { addToBuffers, createWorker, removeFromBuffers } from "../util";
 
+import { Schema } from "@fiftyone/utilities";
 import LRUCache from "lru-cache";
 import { CHUNK_SIZE, MAX_FRAME_CACHE_SIZE_BYTES } from "../constants";
 import { getFrameNumber } from "../elements/util";
@@ -49,6 +49,7 @@ interface AcquireReaderOptions {
   dispatchEvent: (eventType: string, detail: any) => void;
   coloring: Coloring;
   customizeColorSetting: CustomizeColor[];
+  schema: Schema;
 }
 
 const { acquireReader, addFrame } = (() => {
@@ -91,6 +92,7 @@ const { acquireReader, addFrame } = (() => {
     dataset,
     view,
     group,
+    schema,
   }: AcquireReaderOptions): string => {
     streamSize = 0;
     nextRange = [frameNumber, Math.min(frameCount, CHUNK_SIZE + frameNumber)];
@@ -120,11 +122,9 @@ const { acquireReader, addFrame } = (() => {
 
       for (let i = 0; i < frames.length; i++) {
         const frameSample = frames[i];
-        const prefixedFrameSample = Object.fromEntries(
-          Object.entries(frameSample).map(([k, v]) => ["frames." + k, v])
-        );
+        const prefixedFrameSample = withFrames(frameSample);
 
-        const overlays = loadOverlays(prefixedFrameSample);
+        const overlays = loadOverlays(prefixedFrameSample, schema);
         overlays.forEach((overlay) => {
           streamSize += overlay.getSizeBytes();
         });
@@ -165,6 +165,7 @@ const { acquireReader, addFrame } = (() => {
       dataset,
       view,
       group,
+      schema,
     });
     return subscription;
   };
@@ -340,18 +341,16 @@ export class VideoLooker extends AbstractLooker<VideoState, VideoSample> {
       Object.fromEntries(
         Object.entries(sample).filter(([fieldName]) => fieldName !== "frames")
       ),
+      this.state.config.fieldSchema,
       true
     );
 
     const providedFrames = sample.frames?.length
       ? sample.frames
       : [{ frame_number: 1 }];
+
     const providedFrameOverlays = providedFrames.map((frameSample) =>
-      loadOverlays(
-        Object.fromEntries(
-          Object.entries(frameSample).map(([k, v]) => ["frames." + k, v])
-        )
-      )
+      loadOverlays(withFrames(frameSample), this.state.config.fieldSchema)
     );
 
     const frames = providedFrames.map((frameSample, i) => ({
@@ -448,6 +447,7 @@ export class VideoLooker extends AbstractLooker<VideoState, VideoSample> {
       dataset: this.state.config.dataset,
       group: this.state.config.group,
       view: this.state.config.view,
+      schema: this.state.config.fieldSchema,
     });
   }
 
@@ -553,3 +553,8 @@ export class VideoLooker extends AbstractLooker<VideoState, VideoSample> {
     );
   }
 }
+
+const withFrames = <T extends { [key: string]: unknown }>(obj: T): T =>
+  Object.fromEntries(
+    Object.entries(obj).map(([k, v]) => ["frames." + k, v])
+  ) as T;

app/packages/looker/src/overlays/base.ts

@@ -2,6 +2,7 @@
  * Copyright 2017-2023, Voxel51, Inc.
  */
 
+import { getCls } from "@fiftyone/utilities";
 import { BaseState, Coordinates, NONFINITE } from "../state";
 import { getLabelColor, shouldShowLabelTag, sizeBytes } from "./util";
 
@@ -14,7 +15,6 @@ export enum CONTAINS {
 
 export interface BaseLabel {
   id: string;
-  _cls: string;
   frame_number?: number;
   tags: string[];
   index?: number;
@@ -99,6 +99,7 @@ export abstract class CoordinateOverlay<
   }
 
   getColor({
+    config,
     options: {
       coloring,
       customizeColorSetting,
@@ -113,6 +114,7 @@ export abstract class CoordinateOverlay<
       isTagged: shouldShowLabelTag(selectedLabelTags, this.label.tags),
       labelTagColors,
       customizeColorSetting,
+      embeddedDocType: getCls(this.field, config.fieldSchema),
     });
   }