Merge pull request #16 from legoeruro/main

Data modality 1 - Gyroscope

.gitignore

@@ -1,6 +1,9 @@
 # Created by https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode,python,unity
 # Edit at https://www.toptal.com/developers/gitignore?templates=macos,visualstudiocode,python,unity
 
+#additional files
+.grpc_tools
+
 ### macOS ###
 # General
 .DS_Store

protos/arflow/service.proto

@@ -56,6 +56,26 @@ message RegisterRequest {
         float depth_upscale_factor = 2;
     }
     CameraPointCloud camera_point_cloud = 6;
+
+    message CameraPlaneDetection {
+        bool enabled = 1;
+    }
+    CameraPlaneDetection camera_plane_detection = 7;
+
+    message Gyroscope {
+        bool enabled = 1;
+    } 
+    Gyroscope gyroscope = 8;
+
+    message Audio {
+        bool enabled = 1;
+    }
+    Audio audio = 9;
+
+    message Meshing {
+        bool enabled = 1;
+    }
+    Meshing meshing = 10;
 }
 
 message RegisterResponse {
@@ -67,6 +87,40 @@ message DataFrameRequest {
     bytes color = 2;
     bytes depth = 3;
     bytes transform = 4;
+
+    message Vector3 {
+        float x = 1;
+        float y = 2;
+        float z = 3;
+    }
+
+    message Vector2 {
+        float x = 1;
+        float y = 2;
+    }
+    message Planes {
+        Vector3 center = 1;
+        Vector3 normal = 2;
+        Vector2 size = 3;
+    }
+    repeated Planes plane_detection = 5;
+
+    message Quaternion {
+        float x = 1;
+        float y = 2;
+        float z = 3;
+        float w = 4;
+    }
+    message gyroscope_data {
+        Quaternion attitude = 1;
+        Vector3 rotation_rate = 2;
+        Vector3 gravity = 3;
+        Vector3 acceleration = 4;
+    }
+    gyroscope_data gyroscope = 6;
+
+    bytes audio = 8;
+    bytes meshing = 9;
 }
 
 message DataFrameResponse {

python/arflow/core.py

@@ -76,13 +76,13 @@ def data_frame(
 
         if session_configs.camera_color.enabled:
             color_rgb = ARFlowService.decode_rgb_image(session_configs, request.color)
-            decoded_data["color_rgb"] = color_rgb
+            decoded_data["image/color_rgb"] = color_rgb
             color_rgb = np.flipud(color_rgb)
             self.recorder.log("rgb", rr.Image(color_rgb))
 
         if session_configs.camera_depth.enabled:
             depth_img = ARFlowService.decode_depth_image(session_configs, request.depth)
-            decoded_data["depth_img"] = depth_img
+            decoded_data["image/depth_img"] = depth_img
             depth_img = np.flipud(depth_img)
             self.recorder.log("depth", rr.DepthImage(depth_img, meter=1.0))
 
@@ -117,6 +117,56 @@ def data_frame(
             decoded_data["point_cloud_clr"] = clr
             self.recorder.log("world/point_cloud", rr.Points3D(pcd, colors=clr))
 
+        if session_configs.camera_plane_detection.enabled:
+            pass
+
+        if session_configs.gyroscope.enabled:
+            gyro_data = request.gyroscope
+
+            attitude = rr.Quaternion(
+                xyzw=[
+                    gyro_data.attitude.x,
+                    gyro_data.attitude.y,
+                    gyro_data.attitude.z,
+                    gyro_data.attitude.w,
+                ]
+            )
+            rotation_rate = rr.datatypes.Vec3D(
+                [
+                    gyro_data.rotation_rate.x,
+                    gyro_data.rotation_rate.y,
+                    gyro_data.rotation_rate.z,
+                ]
+            )
+            gravity = rr.datatypes.Vec3D(
+                [gyro_data.gravity.x, gyro_data.gravity.y, gyro_data.gravity.z]
+            )
+            acceleration = rr.datatypes.Vec3D(
+                [
+                    gyro_data.acceleration.x,
+                    gyro_data.acceleration.y,
+                    gyro_data.acceleration.z,
+                ]
+            )
+
+            # Attitute is displayed as a box, and the other acceleration variables are displayed as arrows.
+            rr.log(
+                "rotations/gyroscope/attitude",
+                rr.Boxes3D(half_sizes=[0.5, 0.5, 0.5], quaternions=[attitude]),
+            )
+            rr.log(
+                "rotations/gyroscope/rotation_rate",
+                rr.Arrows3D(vectors=[rotation_rate], colors=[[0, 255, 0]]),
+            )
+            rr.log(
+                "rotations/gyroscope/gravity",
+                rr.Arrows3D(vectors=[gravity], colors=[[0, 0, 255]]),
+            )
+            rr.log(
+                "rotations/gyroscope/acceleration",
+                rr.Arrows3D(vectors=[acceleration], colors=[[255, 255, 0]]),
+            )
+
         # Call the for user extension code.
         self.on_frame_received(decoded_data)
 

python/arflow/serve.py

@@ -10,7 +10,7 @@
 
 
 def create_server(
-    service: ARFlowService, port: int = 8500, path_to_save: str | None = "./"
+    service: ARFlowService, port: int = 8500, path_to_save: str | None = None
 ):
     """Run gRPC server."""
     try: