Merge pull request #8 from cake-lab/dev/yiqinzhao

(WIP) Adding the Xihe demo.

.vscode/settings.json

@@ -33,6 +33,7 @@
     ],
     "conventionalCommits.scopes": [
         "server",
-        "client"
+        "client",
+        "others"
     ],
 }

README.md

@@ -12,31 +12,14 @@ Watch our demo video:
 
 Please refer to the individual [server](./python/README.md) and [client](./unity/README.md) installation guides.
 
-## Citation
-
-Please add the following citation in your publication if you used our code for your research project.
-
-```bibtex
-@inproceedings{zhao2024arflow,
-author = {Zhao, Yiqin and Guo, Tian},
-title = {Demo: ARFlow: A Framework for Simplifying AR Experimentation Workflow},
-year = {2024},
-isbn = {9798400704970},
-publisher = {Association for Computing Machinery},
-address = {New York, NY, USA},
-url = {https://doi.org/10.1145/3638550.3643617},
-doi = {10.1145/3638550.3643617},
-abstract = {The recent advancement in computer vision and XR hardware has ignited the community's interest in AR systems research. Similar to traditional systems research, the evaluation of AR systems involves capturing real-world data with AR hardware and iteratively evaluating the targeted system designs [1]. However, it is challenging to conduct scalable and reproducible AR experimentation [2] due to two key reasons. First, there is a lack of integrated framework support in real-world data capturing, which makes it a time-consuming process. Second, AR data often exhibits characteristics, including temporal and spatial variations, and is in a multi-modal format, which makes it difficult to conduct controlled evaluations.},
-booktitle = {Proceedings of the 25th International Workshop on Mobile Computing Systems and Applications},
-pages = {154},
-numpages = {1},
-location = {<conf-loc>, <city>San Diego</city>, <state>CA</state>, <country>USA</country>, </conf-loc>},
-series = {HOTMOBILE '24}
-}
-```
 
 ## Contributors 
 
+<style>
+td { border: none !important; }
+td img { border-radius: 50px; }
+</style>
+
 <!-- readme: contributors -start -->
 <table>
 	<tbody>
@@ -67,6 +50,29 @@ series = {HOTMOBILE '24}
 </table>
 <!-- readme: contributors -end -->
 
+## How to cite ARFlow
+
+Please add the following citation in your publication if you used our code for your research project.
+
+```bibtex
+@inproceedings{zhao2024arflow,
+author = {Zhao, Yiqin and Guo, Tian},
+title = {Demo: ARFlow: A Framework for Simplifying AR Experimentation Workflow},
+year = {2024},
+isbn = {9798400704970},
+publisher = {Association for Computing Machinery},
+address = {New York, NY, USA},
+url = {https://doi.org/10.1145/3638550.3643617},
+doi = {10.1145/3638550.3643617},
+abstract = {The recent advancement in computer vision and XR hardware has ignited the community's interest in AR systems research. Similar to traditional systems research, the evaluation of AR systems involves capturing real-world data with AR hardware and iteratively evaluating the targeted system designs [1]. However, it is challenging to conduct scalable and reproducible AR experimentation [2] due to two key reasons. First, there is a lack of integrated framework support in real-world data capturing, which makes it a time-consuming process. Second, AR data often exhibits characteristics, including temporal and spatial variations, and is in a multi-modal format, which makes it difficult to conduct controlled evaluations.},
+booktitle = {Proceedings of the 25th International Workshop on Mobile Computing Systems and Applications},
+pages = {154},
+numpages = {1},
+location = {<conf-loc>, <city>San Diego</city>, <state>CA</state>, <country>USA</country>, </conf-loc>},
+series = {HOTMOBILE '24}
+}
+```
+
 ## Acknowledgement
 
 This work was supported in part by NSF Grants #2105564 and #2236987, and a VMware grant.

python/examples/demo_xihe/notebooks/export_xihenet.ipynb

@@ -0,0 +1,208 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import torch_cluster"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mps = torch.device(\"mps\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = torch.jit.load('../xihenet.pt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "RecursiveScriptModule(\n",
+       "  original_name=XiheNet\n",
+       "  (sa1): RecursiveScriptModule(\n",
+       "    original_name=PointConvDensitySetAbstraction\n",
+       "    (mlp_convs): RecursiveScriptModule(\n",
+       "      original_name=ModuleList\n",
+       "      (0): RecursiveScriptModule(original_name=Conv2d)\n",
+       "      (1): RecursiveScriptModule(original_name=Conv2d)\n",
+       "    )\n",
+       "    (mlp_bns): RecursiveScriptModule(\n",
+       "      original_name=ModuleList\n",
+       "      (0): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "      (1): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "    )\n",
+       "    (weightnet): RecursiveScriptModule(\n",
+       "      original_name=WeightNet\n",
+       "      (mlp_convs): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (1): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (2): RecursiveScriptModule(original_name=Conv2d)\n",
+       "      )\n",
+       "      (mlp_bns): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (1): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (2): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "      )\n",
+       "    )\n",
+       "    (linear): RecursiveScriptModule(original_name=Linear)\n",
+       "    (bn_linear): RecursiveScriptModule(original_name=BatchNorm1d)\n",
+       "    (densitynet): RecursiveScriptModule(\n",
+       "      original_name=DensityNet\n",
+       "      (mlp_convs): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (1): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (2): RecursiveScriptModule(original_name=Conv2d)\n",
+       "      )\n",
+       "      (mlp_bns): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (1): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (2): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (sa2): RecursiveScriptModule(\n",
+       "    original_name=PointConvDensitySetAbstraction\n",
+       "    (mlp_convs): RecursiveScriptModule(\n",
+       "      original_name=ModuleList\n",
+       "      (0): RecursiveScriptModule(original_name=Conv2d)\n",
+       "      (1): RecursiveScriptModule(original_name=Conv2d)\n",
+       "    )\n",
+       "    (mlp_bns): RecursiveScriptModule(\n",
+       "      original_name=ModuleList\n",
+       "      (0): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "      (1): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "    )\n",
+       "    (weightnet): RecursiveScriptModule(\n",
+       "      original_name=WeightNet\n",
+       "      (mlp_convs): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (1): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (2): RecursiveScriptModule(original_name=Conv2d)\n",
+       "      )\n",
+       "      (mlp_bns): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (1): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (2): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "      )\n",
+       "    )\n",
+       "    (linear): RecursiveScriptModule(original_name=Linear)\n",
+       "    (bn_linear): RecursiveScriptModule(original_name=BatchNorm1d)\n",
+       "    (densitynet): RecursiveScriptModule(\n",
+       "      original_name=DensityNet\n",
+       "      (mlp_convs): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (1): RecursiveScriptModule(original_name=Conv2d)\n",
+       "        (2): RecursiveScriptModule(original_name=Conv2d)\n",
+       "      )\n",
+       "      (mlp_bns): RecursiveScriptModule(\n",
+       "        original_name=ModuleList\n",
+       "        (0): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (1): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "        (2): RecursiveScriptModule(original_name=BatchNorm2d)\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (fc3): RecursiveScriptModule(original_name=Linear)\n",
+       "  (bn3): RecursiveScriptModule(original_name=BatchNorm1d)\n",
+       "  (drop3): RecursiveScriptModule(original_name=Dropout)\n",
+       "  (fc4): RecursiveScriptModule(original_name=Linear)\n",
+       "  (bn4): RecursiveScriptModule(original_name=BatchNorm1d)\n",
+       "  (drop4): RecursiveScriptModule(original_name=Dropout)\n",
+       "  (fc5): RecursiveScriptModule(original_name=Linear)\n",
+       ")"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.eval()\n",
+    "# model.to(mps)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pcd, clr = torch.zeros(1, 3, 1280), torch.zeros(1, 3, 1280)\n",
+    "# pcd, clr = pcd.to(mps), clr.to(mps)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "90.2 ms ± 42.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit\n",
+    "model(pcd, clr)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "rerun_ar",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

python/examples/demo_xihe/scripts/launch_service.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+poetry run python examples/demo_xihe/xihe_server.py

python/examples/demo_xihe/utils3d/__init__.py

@@ -0,0 +1,7 @@
+"""Basic utilities for 3D math operations
+"""
+from .container import *
+from .io import *
+from .math import *
+from .geometry import *
+from . import utils

python/examples/demo_xihe/utils3d/container/__init__.py

@@ -0,0 +1,4 @@
+"""Tools for programming and managing 3D data
+"""
+from .vector import *
+from .matrix import *

python/examples/demo_xihe/utils3d/container/matrix.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import numpy as np
+import utils3d as u3d
+from utils3d.utils.typing import is_sequence_instance
+
+from typing import Sequence
+
+
+class Matrix:
+    _data: np.ndarray
+
+    def __init__(self, source, copy=False):
+        if type(source) is np.ndarray:
+            self._data = np.array(source, copy=copy)
+        elif is_sequence_instance(source, u3d.Vector):
+            self._data = np.concatenate(source)
+
+    @staticmethod
+    def zeros(shape: tuple, dtype=np.float32) -> Matrix:
+        data = np.zeros(shape, dtype=dtype)
+        return Matrix(data)
+
+    def __array__(self) -> np.ndarray:
+        return self._data
+
+    @property
+    def shape(self) -> Sequence[int]:
+        return self._data.shape
+
+    def transpose(self):
+        return self._data.T
+
+
+zeros_matrix = Matrix.zeros
+
+__all__ = ['Matrix', 'zeros_matrix']

python/examples/demo_xihe/utils3d/container/tensor.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import torch
+import numpy as np
+from typing import Tuple
+
+
+class Tensor:
+    _data: np.ndarray
+
+    def __init__(self, source):
+        if isinstance(source, np.ndarray):
+            self._data = source
+
+    @staticmethod
+    def zeros(shape: Tuple) -> Tensor:
+        data = np.zeros(shape)
+        return Tensor(data)
+
+    def __array__(self):
+        return self._data
+
+    def flip_feature_channel(self):
+        self._data = np.moveaxis(self._data, 0, -1)
+
+    def numpy(self):
+        return self._data
+
+    def torch(self):
+        return torch.from_numpy(self._data)
+
+
+zeros_tensor = Tensor.zeros
+
+__all__ = ['Tensor', 'zeros_tensor']