(Experimental) Recording/playback API

src/viser/_viser.py

@@ -23,6 +23,7 @@
 from ._gui_api import GuiApi
 from ._scene_api import SceneApi, cast_vector
 from ._tunnel import ViserTunnel
+from .infra._infra import RecordHandle
 
 
 class _BackwardsCompatibilityShim:
@@ -689,3 +690,14 @@ def send_file_download(
         """
         for client in self.get_clients().values():
             client.send_file_download(filename, content, chunk_size)
+
+    def _start_scene_recording(self) -> RecordHandle:
+        """Start recording outgoing messages for playback or
+        embedding. Includes only the scene.
+
+        **Experimental.** This API may be removed or changed.
+        """
+        return self._websock_server.start_recording(
+            # Don't record GUI messages. This feels brittle.
+            filter=lambda message: "Gui" not in type(message).__name__
+        )

src/viser/client/package.json

@@ -27,7 +27,7 @@
     "immer": "^10.0.4",
     "its-fine": "^1.2.5",
     "mantine-react-table": "^2.0.0-beta.0",
-    "msgpackr": "^1.8.5",
+    "msgpackr": "^1.10.2",
     "postcss": "^8.4.38",
     "prettier": "^3.0.3",
     "react": "^18.2.0",

src/viser/client/src/App.tsx

@@ -34,10 +34,7 @@ import "./index.css";
 import ControlPanel from "./ControlPanel/ControlPanel";
 import { UseGui, useGuiState } from "./ControlPanel/GuiState";
 import { searchParamKey } from "./SearchParamsUtils";
-import {
-  WebsocketMessageProducer,
-  FrameSynchronizedMessageHandler,
-} from "./WebsocketInterface";
+import { WebsocketMessageProducer } from "./WebsocketInterface";
 
 import { Titlebar } from "./Titlebar";
 import { ViserModal } from "./Modal";
@@ -48,8 +45,11 @@ import { useDisclosure } from "@mantine/hooks";
 import { rayToViserCoords } from "./WorldTransformUtils";
 import { ndcFromPointerXy, opencvXyFromPointerXy } from "./ClickUtils";
 import { theme } from "./AppTheme";
+import { FrameSynchronizedMessageHandler } from "./MessageHandler";
+import { PlaybackFromFile } from "./FilePlayback";
 
 export type ViewerContextContents = {
+  messageSource: "websocket" | "file_playback";
   // Zustand hooks.
   useSceneTree: UseSceneTree;
   useGui: UseGui;
@@ -130,8 +130,15 @@ function ViewerRoot() {
   const initialServer =
     servers.length >= 1 ? servers[0] : getDefaultServerFromUrl();
 
+  // Playback mode for embedding viser.
+  const playbackPath = new URLSearchParams(window.location.search).get(
+    "playbackPath",
+  );
+  console.log(playbackPath);
+
   // Values that can be globally accessed by components in a viewer.
   const viewer: ViewerContextContents = {
+    messageSource: playbackPath === null ? "websocket" : "file_playback",
     useSceneTree: useSceneTreeState(),
     useGui: useGuiState(initialServer),
     websocketRef: React.useRef(null),
@@ -169,7 +176,12 @@ function ViewerRoot() {
 
   return (
     <ViewerContext.Provider value={viewer}>
-      <WebsocketMessageProducer />
+      {viewer.messageSource === "websocket" ? (
+        <WebsocketMessageProducer />
+      ) : null}
+      {viewer.messageSource === "file_playback" ? (
+        <PlaybackFromFile fileUrl={playbackPath!} />
+      ) : null}
       <ViewerContents />
     </ViewerContext.Provider>
   );
@@ -235,11 +247,14 @@ function ViewerContents() {
               <ViewerCanvas>
                 <FrameSynchronizedMessageHandler />
               </ViewerCanvas>
-              {viewer.useGui((state) => state.theme.show_logo) ? (
+              {viewer.useGui((state) => state.theme.show_logo) &&
+              viewer.messageSource == "websocket" ? (
                 <ViserLogo />
               ) : null}
             </Box>
-            <ControlPanel control_layout={control_layout} />
+            {viewer.messageSource == "websocket" ? (
+              <ControlPanel control_layout={control_layout} />
+            ) : null}
           </Box>
         </Box>
       </MantineProvider>

src/viser/client/src/FilePlayback.tsx

@@ -0,0 +1,81 @@
+import { unpack } from "msgpackr";
+import { Message } from "./WebsocketMessages";
+import React, { useContext, useEffect } from "react";
+import { ViewerContext } from "./App";
+
+interface SerializedMessages {
+  loopStartIndex: number | null;
+  durationSeconds: number;
+  messages: [number, Message][];
+}
+
+async function deserializeMsgpackFile<T>(fileUrl: string): Promise<T> {
+  // Fetch the file using fetch()
+  const response = await fetch(fileUrl);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch the file: ${response.statusText}`);
+  }
+  return new Promise<T>((resolve) => {
+    let length = 0;
+    const buffer: Uint8Array[] = [];
+    response.body!.pipeThrough(new DecompressionStream("gzip")).pipeTo(
+      new WritableStream<Uint8Array>({
+        write(chunk) {
+          buffer.push(chunk);
+          length += chunk.length;
+        },
+        close() {
+          const output = new Uint8Array(length);
+          let offset = 0;
+          for (const chunk of buffer) {
+            output.set(chunk, offset);
+            offset += chunk.length;
+          }
+          console.log(output.length);
+          resolve(unpack(output));
+        },
+        abort(err) {
+          console.error("Stream aborted:", err);
+        },
+      }),
+    );
+  });
+}
+
+export function PlaybackFromFile({ fileUrl }: { fileUrl: string }) {
+  const viewer = useContext(ViewerContext)!;
+  const messageQueueRef = viewer.messageQueueRef;
+
+  useEffect(() => {
+    deserializeMsgpackFile<SerializedMessages>(fileUrl).then((recording) => {
+      let messageIndex = 0;
+
+      function continuePlayback() {
+        const currentTimeSeconds = recording.messages[messageIndex][0];
+        while (currentTimeSeconds >= recording.messages[messageIndex][0]) {
+          messageQueueRef.current.push(recording.messages[messageIndex][1]);
+          messageIndex += 1;
+
+          // Either finish playback or loop.
+          if (messageIndex === recording.messages.length) {
+            if (recording.loopStartIndex === null) return;
+            messageIndex = recording.loopStartIndex;
+            setTimeout(
+              continuePlayback,
+              (recording.durationSeconds - currentTimeSeconds) * 1000.0,
+            );
+            return;
+          }
+        }
+
+        // Handle next set of frames.
+        setTimeout(
+          continuePlayback,
+          (recording.messages[messageIndex][0] - currentTimeSeconds) * 1000.0,
+        );
+      }
+      setTimeout(continuePlayback, recording.messages[0][0] * 1000.0);
+    });
+  });
+  return <></>;
+}