diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 89efb71..6b3d0fa 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -10,6 +10,7 @@ on: - 'python/**' - '.github/workflows/website.yml' - 'website/**' + - 'unity/**' # Alternative: only build for tags. # tags: @@ -23,7 +24,30 @@ permissions: contents: read jobs: - build-docs: + build-client-docs-as-artifact: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Install docfx + run: dotnet tool install -g docfx + + - name: Run script to build the documentation + working-directory: ./unity/Documentation + run: ./scripts/build.cmd + + # - name: Move docs to website directory + # run: | + # mkdir -p ./website/docs/client/ + # cp -r ./unity/Documentation/clientHTMLOutput/* ./website/docs/client/ + # Upload the website directory as an artifact + - uses: actions/upload-artifact@v4 + with: + name: client-docs + path: ./unity/Documentation/clientHTMLOutput + + build-server-docs: + needs: build-client-docs-as-artifact runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -63,13 +87,28 @@ jobs: mkdir -p ./website/docs/server/ cp -r ./python/docs/* ./website/docs/server/ + # Get client docs to use as part of pages artifact + - uses: actions/download-artifact@v4 + with: + name: client-docs + path: ./website/docs/client + + # # cleanup client docs artifacts + # - name: Delete client docs artifact + # run: | + # github.rest.actions.deleteArtifact({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # artifact_id: ${{ steps.artifact-download.outputs.artifact-id }} + # }); + - uses: actions/upload-pages-artifact@v3 with: path: ./website # Single deploy job since we're just deploying deploy: - needs: build-docs + needs: build-server-docs runs-on: ubuntu-latest permissions: pages: write diff --git a/.gitignore b/.gitignore index 6b7445d..5280b65 100644 --- a/.gitignore +++ b/.gitignore @@ -242,6 +242,7 @@ ExportedObj/ .consulo/ *.csproj *.unityproj +# add .sln files to build documentation *.sln *.suo *.tmp diff --git a/README.md b/README.md index 0334d51..973b372 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,13 @@ series = {HOTMOBILE '24} Yiqin Zhao + + + legoeruro +
+ Khang Luu + + FelixNgFender diff --git a/unity/.gitignore b/unity/.gitignore index efb8fca..50878a6 100644 --- a/unity/.gitignore +++ b/unity/.gitignore @@ -34,6 +34,7 @@ ExportedObj/ .consulo/ *.csproj *.unityproj +# solution file is uploaded for doc building *.sln *.suo *.tmp diff --git a/unity/Assets/Scripts/ARFlow/ARFlowClient.cs b/unity/Assets/Scripts/ARFlow/ARFlowClient.cs index 06f8f4f..737c970 100644 --- a/unity/Assets/Scripts/ARFlow/ARFlowClient.cs +++ b/unity/Assets/Scripts/ARFlow/ARFlowClient.cs @@ -5,12 +5,20 @@ namespace ARFlow { + /// + /// This class represent the implementation for the client, using the gRPC protocol generated by Protobuf. + /// The client of ARFlow allows registering to the server and sending data frames to the server. + /// public class ARFlowClient { private readonly GrpcChannel _channel; private readonly ARFlowService.ARFlowServiceClient _client; private string _sessionId; + /// + /// Initialize the client + /// + /// The address (AKA server URL) to connect to public ARFlowClient(string address) { var handler = new YetAnotherHttpHandler() { Http2Only = true }; @@ -27,6 +35,10 @@ public ARFlowClient(string address) _channel.Dispose(); } + /// + /// Connect to the server with a request that contain register data of about the camera. + /// + /// Register data (AKA metadata) of the camera. The typing of this is generated by Protobuf. public void Connect(RegisterRequest requestData) { try @@ -44,6 +56,10 @@ public void Connect(RegisterRequest requestData) } } + /// + /// Send a data of a frame to the server. + /// + /// Data of the frame. The typing of this is generated by Protobuf. public void SendFrame(DataFrameRequest frameData) { frameData.Uid = _sessionId; diff --git a/unity/Assets/Scripts/ARFlow/XRCpuImageExt.cs b/unity/Assets/Scripts/ARFlow/XRCpuImageExt.cs index c32db74..9cd64e8 100644 --- a/unity/Assets/Scripts/ARFlow/XRCpuImageExt.cs +++ b/unity/Assets/Scripts/ARFlow/XRCpuImageExt.cs @@ -5,20 +5,33 @@ namespace ARFlow { + /// + /// Interface for encoding CPU image + /// internal interface IXRCpuImageEncodable { public byte[] Encode(); } + /// + /// Depth image information. + /// internal struct XRDepthImage : IXRCpuImageEncodable { private XRCpuImage _image; + /// + /// Get Depth image from AROcclusionManager. + /// public XRDepthImage(AROcclusionManager occlusionManager) { occlusionManager.TryAcquireEnvironmentDepthCpuImage(out _image); } + /// + /// Encode depth image + /// + /// Depth image in bytes public byte[] Encode() { return _image.GetPlane(0).data.ToArray(); @@ -30,6 +43,9 @@ public void Dispose() } } + /// + /// Depth image information with confidence + /// internal struct XRConfidenceFilteredDepthImage : IXRCpuImageEncodable { private XRCpuImage _depthImage; @@ -41,6 +57,11 @@ public Vector2Int Size() return _depthImage.dimensions; } + /// + /// Get depth and depth confidence from AROcclusionManager. + /// + /// + /// Min confidence for filtering public XRConfidenceFilteredDepthImage(AROcclusionManager occlusionManager, int minConfidence = 1) { // occlusionManager.TryAcquireEnvironmentDepthCpuImage(out _depthImage); @@ -49,6 +70,12 @@ public XRConfidenceFilteredDepthImage(AROcclusionManager occlusionManager, int m _minConfidence = minConfidence; } + /// + /// For each depth value, if confidence is lower than minConfidence, it will be ignored (replaced with 0s). + /// The rest is encoded to bytes. + /// + /// + /// Encoded bytes public byte[] Encode() { var depthValues = _depthImage.GetPlane(0).data.ToArray(); @@ -83,6 +110,9 @@ public void Dispose() } } + /// + /// Color image information + /// internal struct XRYCbCrColorImage : IXRCpuImageEncodable { private XRCpuImage _image; @@ -91,7 +121,11 @@ internal struct XRYCbCrColorImage : IXRCpuImageEncodable private readonly Vector2Int _nativeSize; private readonly Vector2Int _sampleSize; - + /// + /// Get image from ARCameraManager, and set scale to relative of sample (depth) size. + /// + /// + /// public XRYCbCrColorImage(ARCameraManager cameraManager, Vector2Int sampleSize) { cameraManager.TryAcquireLatestCpuImage(out _image); @@ -101,6 +135,10 @@ public XRYCbCrColorImage(ARCameraManager cameraManager, Vector2Int sampleSize) _scale = _sampleSize.x / (float)_nativeSize.x; } + /// + /// Resample color image to right size and convert to bytes. + /// + /// Encoded bytes public byte[] Encode() { var size = _sampleSize.x * _sampleSize.y + 2 * (_sampleSize.x / 2 * _sampleSize.y / 2); diff --git a/unity/Assets/Scripts/ARFlowDeviceSample.cs b/unity/Assets/Scripts/ARFlowDeviceSample.cs index 691eda7..76831e2 100644 --- a/unity/Assets/Scripts/ARFlowDeviceSample.cs +++ b/unity/Assets/Scripts/ARFlowDeviceSample.cs @@ -8,7 +8,13 @@ public class ARFlowDeviceSample : MonoBehaviour { + /// + /// Camera image data's manager from the device camera + /// public ARCameraManager cameraManager; + /// + /// Depth data's manager from the device camera + /// public AROcclusionManager occlusionManager; public Button connectButton; @@ -37,7 +43,10 @@ void Start() // Application.targetFrameRate = 30; } - + /// + /// Get register request data from camera and send to server. + /// Image and depth info is acquired once to get information for the request, and is disposed afterwards. + /// private void OnConnectButtonClick() { try @@ -98,6 +107,10 @@ private void OnConnectButtonClick() } } + /// + /// On pause, pressing the button changes the _enabled flag to true (and text display) and data starts sending in Update() + /// On start, pressing the button changes the _enabled flag to false and data stops sending + /// private void OnStartPauseButtonClick() { Debug.Log($"Current framerate: {Application.targetFrameRate}"); @@ -113,6 +126,10 @@ void Update() UploadFrame(); } + /// + /// Get color image and depth information, and copy camera's transform from float to bytes. + /// This data is sent over the server. + /// private void UploadFrame() { var colorImage = new XRYCbCrColorImage(cameraManager, _sampleSize); diff --git a/unity/Assets/Scripts/ARFlowKuafuData.cs b/unity/Assets/Scripts/ARFlowKuafuData.cs deleted file mode 100644 index 3f2267c..0000000 --- a/unity/Assets/Scripts/ARFlowKuafuData.cs +++ /dev/null @@ -1,139 +0,0 @@ -using System; -using ARFlow; -using Google.Protobuf; -using TMPro; -using UnityEngine; -using UnityEngine.UI; -using UnityEngine.XR.ARFoundation; - -public class ARFlowKuafuData : MonoBehaviour -{ - public ARCameraManager cameraManager; - public AROcclusionManager occlusionManager; - - public Button connectButton; - public Button triggerButton; - - private ARFlowClient _client; - private Vector2Int _sampleSize; - private bool _enabled = false; - - // Start is called before the first frame update - void Start() - { - // const string serverURL = "http://192.168.1.100:8500"; - // const string serverURL = "http://169.254.189.74:8500"; - // const string serverURL = "http://100.71.197.137:8500"; - const string serverURL = "http://192.168.1.139:8500"; - _client = new ARFlowClient(serverURL); - - connectButton.onClick.AddListener(OnConnectButtonClick); - triggerButton.onClick.AddListener(OnTriggerButtonClick); - - // QualitySettings.vSyncCount = 0; - // Application.targetFrameRate = 30; - } - - - private void OnConnectButtonClick() - { - try - { - cameraManager.TryGetIntrinsics(out var k); - cameraManager.TryAcquireLatestCpuImage(out var colorImage); - occlusionManager.TryAcquireEnvironmentDepthCpuImage(out var depthImage); - - _sampleSize = depthImage.dimensions; - - var requestData = new RegisterRequest() - { - DeviceName = SystemInfo.deviceName, - CameraIntrinsics = new RegisterRequest.Types.CameraIntrinsics() - { - FocalLengthX = k.focalLength.x, - FocalLengthY = k.focalLength.y, - PrincipalPointX = k.principalPoint.x, - PrincipalPointY = k.principalPoint.y, - ResolutionX = k.resolution.x, - ResolutionY = k.resolution.y, - }, - CameraColor = new RegisterRequest.Types.CameraColor() - { - Enabled = true, - DataType = "YCbCr420", - ResizeFactorX = depthImage.dimensions.x / (float)colorImage.dimensions.x, - ResizeFactorY = depthImage.dimensions.x / (float)colorImage.dimensions.x, - }, - CameraDepth = new RegisterRequest.Types.CameraDepth() - { - Enabled = true, - DataType = "f32", // Float32 for iOS, UInt16 for Android - ConfidenceFilteringLevel = 0, - ResolutionX = depthImage.dimensions.x, - ResolutionY = depthImage.dimensions.y - }, - CameraTransform = new RegisterRequest.Types.CameraTransform() - { - Enabled = true - }, - CameraPointCloud = new RegisterRequest.Types.CameraPointCloud() - { - Enabled = true, - DepthUpscaleFactor = 1.0f, - }, - }; - colorImage.Dispose(); - depthImage.Dispose(); - - _client.Connect(requestData); - } - catch (System.Exception e) - { - Debug.LogError(e); - } - } - - private void OnTriggerButtonClick() - { - // _enabled = !_enabled; - // triggerButton.GetComponentInChildren().text = _enabled ? "Pause" : "Start"; - - // Debug.Log($"Current framerate: {Application.targetFrameRate}"); - - UploadFrame(); - } - - // Update is called once per frame - void Update() - { - // if (!_enabled) return; - // UploadFrame(); - } - - private void UploadFrame() - { - var colorImage = new XRYCbCrColorImage(cameraManager, _sampleSize); - var depthImage = new XRConfidenceFilteredDepthImage(occlusionManager, 0); - - const int transformLength = 3 * 4 * sizeof(float); - var m = Camera.main!.transform.localToWorldMatrix; - var cameraTransformBytes = new byte[transformLength]; - - Buffer.BlockCopy(new[] - { - m.m00, m.m01, m.m02, m.m03, - m.m10, m.m11, m.m12, m.m13, - m.m20, m.m21, m.m22, m.m23 - }, 0, cameraTransformBytes, 0, transformLength); - - _client.SendFrame(new DataFrameRequest() - { - Color = ByteString.CopyFrom(colorImage.Encode()), - Depth = ByteString.CopyFrom(depthImage.Encode()), - Transform = ByteString.CopyFrom(cameraTransformBytes) - }); - - colorImage.Dispose(); - depthImage.Dispose(); - } -} diff --git a/unity/Assets/Scripts/ARFlowKuafuData.cs.meta b/unity/Assets/Scripts/ARFlowKuafuData.cs.meta deleted file mode 100644 index 050993a..0000000 --- a/unity/Assets/Scripts/ARFlowKuafuData.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 7cdb79e2c2c5e4b268302b4d29d84a82 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/unity/Assets/Scripts/ARFlowMockDataSample.cs b/unity/Assets/Scripts/ARFlowMockDataSample.cs index d82c823..71583f3 100644 --- a/unity/Assets/Scripts/ARFlowMockDataSample.cs +++ b/unity/Assets/Scripts/ARFlowMockDataSample.cs @@ -4,6 +4,10 @@ using UnityEngine.UI; using TMPro; +/// +/// Class for sending mock data to the server. +/// Used in the MockData scene. +/// public class ARFlowMockDataSample : MonoBehaviour { public TMP_InputField addressInput; @@ -11,6 +15,9 @@ public class ARFlowMockDataSample : MonoBehaviour public Button sendButton; private ARFlowClient _client; + /// + /// Size of mock data generated to send to server, in width (x) and length (y). + /// private Vector2Int _sampleSize; private System.Random _rnd = new System.Random(); @@ -21,6 +28,10 @@ void Start() sendButton.onClick.AddListener(OnSendButtonClick); } + /// + /// On connection, send register request with mock camera's register data. + /// For the mock sample, we are only sending color data. + /// private void OnConnectButtonClick() { string serverURL = addressInput.text; @@ -58,6 +69,9 @@ private void OnConnectButtonClick() }); } + /// + /// On pressing send, 1 frame of mock data in bytes is generated from System.Random and sended. + /// private void OnSendButtonClick() { var size = _sampleSize.x * _sampleSize.y + 2 * (_sampleSize.x / 2 * _sampleSize.y / 2); diff --git a/unity/Assets/Scripts/ARFlowUnityDataSample.cs b/unity/Assets/Scripts/ARFlowUnityDataSample.cs index 2971591..cd19ce8 100644 --- a/unity/Assets/Scripts/ARFlowUnityDataSample.cs +++ b/unity/Assets/Scripts/ARFlowUnityDataSample.cs @@ -18,12 +18,34 @@ public class ARFlowUnityDataSample : MonoBehaviour private ARFlowClient _client; private Vector2Int _sampleSize; private System.Random _rnd = new System.Random(); + + + /// + /// Camera to exist in the Unity Scene, of which we will capture data from + /// private Camera _captureCamera; - private Shader _depthShader; - private Texture2D _colorTexture; + + /// + /// Camera color is rendered into this texture + /// private RenderTexture _colorRenderTexture; - private Texture2D _depthTexture; + /// + /// Read pixels of the _colorRenderTexture to get color data + /// + private Texture2D _colorTexture; + + /// + /// Shader to render depth from camera. + /// + private Shader _depthShader; + /// + /// Camera depth is rendered into this texture + /// private RenderTexture _depthRenderTexture; + /// + /// Read pixels of the _depthTexture to get depth data + /// + private Texture2D _depthTexture; // Start is called before the first frame update void Start() @@ -45,6 +67,9 @@ void Start() Application.targetFrameRate = 60; } + /// + /// On connection, calculate register request data and send to server. + /// private void OnConnectButtonClick() { Matrix4x4 projectionMatrix = _captureCamera.projectionMatrix; @@ -94,12 +119,19 @@ private void OnConnectButtonClick() }); } + /// + /// On pause, pressing the button changes the _enabled flag to true (and text display) and data starts sending in Update() + /// On start, pressing the button changes the _enabled flag to false and data stops sending + /// private void OnStartPauseButtonClick() { _enabled = !_enabled; startPauseButton.GetComponentInChildren().text = _enabled ? "Pause" : "Start"; } + /// + /// Render RGB and depth data to a Texture2D and read raw bytes data from the Texture2D. This data is sent over the server. + /// private void UploadFrame() { // Render RGB. @@ -131,6 +163,9 @@ private void UploadFrame() } // Update is called once per frame + /// + /// If the _enabled flag is true, starts uploading frame to server. + /// void Update() { if (!_enabled) return; diff --git a/unity/Documentation/.gitignore b/unity/Documentation/.gitignore new file mode 100644 index 0000000..06344e9 --- /dev/null +++ b/unity/Documentation/.gitignore @@ -0,0 +1,6 @@ +# ignore the API and client directory since it will be generated by docfx. +/api + +# Client docs directory inside documentation is for local environment. +# When running github workflows the files will be moved into the website/docs directory (in root) +/clientHTMLOutput \ No newline at end of file diff --git a/unity/Documentation/README.md b/unity/Documentation/README.md new file mode 100644 index 0000000..e7b4873 --- /dev/null +++ b/unity/Documentation/README.md @@ -0,0 +1,5 @@ +# Build client documentation with Docfx +To build client docs with Docfx, make sure you have dotnet 6 and docfx installed. +Run either of the scripts inside /scripts with the current working directory being the /Documentation folder. + +The documentation HTML is outputted into the /client folder. \ No newline at end of file diff --git a/unity/Documentation/docfx.json b/unity/Documentation/docfx.json new file mode 100644 index 0000000..06e3ca2 --- /dev/null +++ b/unity/Documentation/docfx.json @@ -0,0 +1,31 @@ +{ + "metadata": [ + { + "src": [ + { + "src": "../Assets/Scripts", + "files": ["**/*.cs"] + } + ], + "dest": "api", + "filter": "filterConfig.yml", + "allowCompilationErrors": true + } + ], + "build": { + "content": [ + { + "files": ["**/*.{md,yml}"], + "exclude": ["_site/**"] + } + ], + "output": "./clientHTMLOutput", + "template": ["default", "modern"], + "globalMetadata": { + "_appName": "ARFlow", + "_appTitle": "ARFlow", + "_enableSearch": true, + "pdf": true + } + } +} diff --git a/unity/Documentation/filterConfig.yml b/unity/Documentation/filterConfig.yml new file mode 100644 index 0000000..c77f2b8 --- /dev/null +++ b/unity/Documentation/filterConfig.yml @@ -0,0 +1,38 @@ +### YamlMime:ManagedReference +apiRules: +- include: # The namespaces to generate + uidRegex: ^ARFlow + type: Namespace +- exclude: + uidRegex: .* # Every other namespaces are ignored + type: Namespace + +#Use something more specific like this if we want to include documentation for monobehavior classes +# apiRules: +# - exclude: +# uidRegex: ^System\.Object +# type: Type +# - exclude: +# uidRegex: ^System\.ValueType +# type: Type +# - exclude: +# uidRegex: ^System\.Attribute +# type: Type +# - exclude: +# uidRegex: ^UnityEngine\.MonoBehaviour +# type: Type +# - exclude: +# uidRegex: ^UnityEngine\.Behaviour +# type: Type +# - exclude: +# uidRegex: ^UnityEngine\.Component +# type: Type +# - exclude: +# uidRegex: ^UnityEngine\.Object +# type: Type +# - exclude: +# uidRegex: ^UnityEngine\.ScriptableObject +# type: Type +# - exclude: +# uidRegex: ^UnityEditor\.Editor +# type: Type \ No newline at end of file diff --git a/unity/Documentation/index.md b/unity/Documentation/index.md new file mode 100644 index 0000000..54ba5ef --- /dev/null +++ b/unity/Documentation/index.md @@ -0,0 +1,24 @@ +# ARFlow Client + +The ARFlow client is responsible for on-device AR data collection and high-performance AR data streaming. We implement the ARFlow client as a Unity application that can be easily ported to different platforms and devices. + +The core functions are implemented in `unity/Assets/Scripts/ARFlow`. We show three example ARFlow integration of three different data sources: + +- Mock data: inside ./Assets/Scripts/ARFlowMockDataSample.cs +- ARFoundation device data: inside ./Assets/Scripts/ARFlowDeviceSample.cs +- Unity scene data: inside ./Assets/Scripts/ARFlowUnityDataSample.cs + +To use ARFlow with your own device, you should directly deploy our client code to your AR device. +Please compile the Unity code for your target deployment platform and install the compiled application. + +Currently, we support the following platforms: + +- iOS (iPhone, iPad) +- Android (Android Phone) + + + + +## Document generation +This document for Unity/C# solution code was generated by a tool called Docfx. +To get started on building the document, install Docfx from Dotnet and run the build.cmd script. Note that you will have to be on Windows. \ No newline at end of file diff --git a/unity/Documentation/scripts/build.cmd b/unity/Documentation/scripts/build.cmd new file mode 100644 index 0000000..cf7a81d --- /dev/null +++ b/unity/Documentation/scripts/build.cmd @@ -0,0 +1,22 @@ +:: Borrowed from https://github.com/open-telemetry/opentelemetry-dotnet +SETLOCAL +SETLOCAL ENABLEEXTENSIONS + +rmdir /s /q api +rmdir /s /q clientHTMLOutput + + + +docfx metadata +docfx build docfx.json > docfx.log +@IF NOT %ERRORLEVEL% == 0 ( + type docfx.log + ECHO Error: docfx build failed. 1>&2 + @REM EXIT /B %ERRORLEVEL% +) +@type docfx.log +@type docfx.log | findstr /C:"Build succeeded." +@IF NOT %ERRORLEVEL% == 0 ( + ECHO There are build warnings. 1>&2 + @REM EXIT /B %ERRORLEVEL% +) \ No newline at end of file diff --git a/unity/Documentation/scripts/build.sh b/unity/Documentation/scripts/build.sh new file mode 100644 index 0000000..44770a9 --- /dev/null +++ b/unity/Documentation/scripts/build.sh @@ -0,0 +1,16 @@ +rm -rf clientHTMLOutput +rm -rf api + +docfx metadata +docfx build docfx.json > docfx.log + +if [ $? -ne 0 ]; then +cat docfx.log +echo "Error: docfx build failed." >&2 +fi + +cat docfx.log + +if ! grep -q "Build succeeded." docfx.log; then +echo "There are build warnings." >&2 +fi diff --git a/unity/Documentation/toc.yml b/unity/Documentation/toc.yml new file mode 100644 index 0000000..a2a2fa2 --- /dev/null +++ b/unity/Documentation/toc.yml @@ -0,0 +1,2 @@ +- name: API + href: api/ \ No newline at end of file