feat(WORK IN PROGRESS): Add ArmNN and rknn2 detectors. Add docs about orange pi 5

Dockerfile

@@ -178,6 +178,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility"
 
 ENV PATH="/usr/lib/btbn-ffmpeg/bin:/usr/local/go2rtc/bin:/usr/local/nginx/sbin:${PATH}"
 
+# Install mpp empowerred ffmpeg for arm64
+RUN --mount=type=tmpfs,target=/tmp --mount=type=tmpfs,target=/var/cache/apt \
+    --mount=type=bind,source=docker/mpp.sh,target=/deps/mpp.sh \
+    /deps/mpp.sh
+
 # Install dependencies
 RUN --mount=type=bind,source=docker/install_deps.sh,target=/deps/install_deps.sh \
     /deps/install_deps.sh

docker/build_nginx.sh

@@ -9,11 +9,12 @@ RTMP_MODULE_VERSION="1.2.1"
 
 cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list
 sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list
-apt-get update
+apt-get -qq update
 
 apt-get -yqq build-dep nginx
 
-apt-get -yqq install --no-install-recommends ca-certificates wget
+# TODO: move all apt-get installs to dedicated script for having it in a single Docker layer = performance of reruns.
+apt-get -yqq install --no-install-recommends ca-certificates wget libaio-dev
 update-ca-certificates -f
 mkdir /tmp/nginx
 wget -nv https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz
@@ -52,15 +53,21 @@ rm v${RTMP_MODULE_VERSION}.tar.gz
 
 cd /tmp/nginx
 
+ARCH=$(uname -m)
+CC_OPT="-O3 -Wno-error=implicit-fallthrough"
+
+if [ "$ARCH" = "aarch64" ]; then
+    CC_OPT="${CC_OPT} -march=armv8-a+crc -mfpu=neon-fp-armv8 -mfloat-abi=hard"
+fi
+
 ./configure --prefix=/usr/local/nginx \
-    --with-file-aio \
     --with-http_sub_module \
     --with-http_ssl_module \
     --with-threads \
     --add-module=../nginx-vod-module \
     --add-module=../nginx-secure-token-module \
     --add-module=../nginx-rtmp-module \
-    --with-cc-opt="-O3 -Wno-error=implicit-fallthrough"
+    --with-cc-opt="${CC_OPT}"
 
 make -j$(nproc) && make install
 rm -rf /usr/local/nginx/html /usr/local/nginx/conf/*.default

docker/install_deps.sh

@@ -4,6 +4,7 @@ set -euxo pipefail
 
 apt-get -qq update
 
+# TODO: move all apt-get installs to dedicated script for having it in a single Docker layer = performance of reruns.
 apt-get -qq install --no-install-recommends -y \
     apt-transport-https \
     gnupg \
@@ -47,13 +48,22 @@ if [[ "${TARGETARCH}" == "arm" ]]; then
 fi
 
 # ffmpeg -> arm64
+if command -v ffmpeg >/dev/null 2>&1; then
+    HAS_FFMPEG=1
+else
+    HAS_FFMPEG=0
+fi
+
+if [[ "${HAS_FFMPEG}" == 0 ]]; then
 if [[ "${TARGETARCH}" == "arm64" ]]; then
     # add raspberry pi repo
     gpg --no-default-keyring --keyring /usr/share/keyrings/raspbian.gpg --keyserver keyserver.ubuntu.com --recv-keys 82B129927FA3303E
     echo "deb [signed-by=/usr/share/keyrings/raspbian.gpg] https://archive.raspberrypi.org/debian/ bullseye main" | tee /etc/apt/sources.list.d/raspi.list
     apt-get -qq update
     apt-get -qq install --no-install-recommends --no-install-suggests -y ffmpeg
 fi
+fi
+
 
 # arch specific packages
 if [[ "${TARGETARCH}" == "amd64" ]]; then

docker/mpp.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+apt-get -qq update
+apt install -y pkg-config
+
+if [ -e /usr/local/include/mpp/mpp.h ] || [ -e /usr/include/mpp/mpp.h ] || pkg-config --exists rockchip-mpp; then
+    HAS_MPP=1
+else
+# Install mpp from sources
+apt-get -qq update
+# TODO: move all apt-get installs to dedicated script for having it in a single Docker layer = performance of reruns.
+apt-get install -y git build-essential yasm pkg-config \
+libtool coreutils autoconf automake build-essential cmake \
+doxygen git graphviz imagemagick libasound2-dev libass-dev \
+libavcodec-dev libavdevice-dev libavfilter-dev libavformat-dev \
+libavutil-dev libfreetype6-dev libgmp-dev libmp3lame-dev \
+libopencore-amrnb-dev libopencore-amrwb-dev libopus-dev \
+librtmp-dev libsdl2-dev libsdl2-image-dev libsdl2-mixer-dev \
+libsdl2-net-dev libsdl2-ttf-dev libsnappy-dev libsoxr-dev \
+libssh-dev libssl-dev libtool libv4l-dev libva-dev libvdpau-dev \
+libvo-amrwbenc-dev libvorbis-dev libwebp-dev libx264-dev libx265-dev \
+libxcb-shape0-dev libxcb-shm0-dev libxcb-xfixes0-dev libxcb1-dev \
+libxml2-dev lzma-dev meson nasm pkg-config python3-dev \
+python3-pip texinfo wget yasm zlib1g-dev libdrm-dev libaom-dev libdav1d-dev \
+libmp3lame-dev
+
+cd /tmp
+git clone https://github.com/rockchip-linux/mpp.git
+cd mpp
+mkdir build || true && cd build
+
+ARCH=$(uname -m)
+EXTRA_CFLAGS=""
+EXTRA_CXXFLAGS=""
+
+if [ "$ARCH" = "aarch64" ]; then
+    EXTRA_CFLAGS="-march=armv8-a+crc -mfpu=neon-fp-armv8 -mfloat-abi=hard"
+    EXTRA_CXXFLAGS="-march=armv8-a+crc -mfpu=neon-fp-armv8 -mfloat-abi=hard"
+fi
+
+cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_C_FLAGS="${EXTRA_CFLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_CXXFLAGS}" ../
+make -j$(nproc)
+make install
+ldconfig
+
+fi
+
+# TODO: consider moving ffmpeg compillation to a dedicated ffmpeg.sh script
+if command -v ffmpeg >/dev/null 2>&1; then
+    HAS_FFMPEG=1
+else
+    HAS_FFMPEG=0
+fi
+
+if [[ "${HAS_FFMPEG}" == 1 ]]; then
+# To avoid possible race condition.
+    apt -y remove ffmpeg
+fi
+# Compile ffmpeg
+
+cd /tmp
+git clone https://github.com/FFmpeg/FFmpeg.git
+cd FFmpeg
+
+ARCH=$(uname -m)
+EXTRA_CFLAGS="-I/usr/local/include"
+EXTRA_LDFLAGS="-L/usr/local/lib"
+
+if [ "$ARCH" = "aarch64" ]; then
+    EXTRA_CFLAGS="${EXTRA_CFLAGS} -march=armv8-a+crc -mfpu=neon-fp-armv8 -mfloat-abi=hard"
+fi
+
+PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" ./configure \
+    --enable-rkmpp \
+    --extra-cflags="${EXTRA_CFLAGS}" \
+    --extra-ldflags="${EXTRA_LDFLAGS}" \
+    --extra-libs="-lpthread -lm -latomic" \
+    --arch=arm64 \
+    --enable-gmp \
+    --enable-gpl \
+    --enable-libaom \
+    --enable-libass \
+    --enable-libdav1d \
+    --enable-libdrm \
+    --enable-libfreetype \
+    --enable-libmp3lame \
+    --enable-libopencore-amrnb \
+    --enable-libopencore-amrwb \
+    --enable-libopus \
+    --enable-librtmp \
+    --enable-libsnappy \
+    --enable-libsoxr \
+    --enable-libssh \
+    --enable-libvorbis \
+    --enable-libzimg \
+    --enable-libwebp \
+    --enable-libx264 \
+    --enable-libx265 \
+    --enable-libxml2 \
+    --enable-nonfree \
+    --enable-version3 \
+    --target-os=linux \
+    --enable-pthreads \
+    --enable-openssl \
+    --enable-hardcoded-tables
+
+make -j$(nproc)
+make install
+ldconfig
+
+cd /tmp
+rm -rf mpp
+rm -rf FFmpeg

docs/docs/configuration/detectors.md

@@ -138,6 +138,29 @@ model:
   labelmap_path: /path/to/coco_80cl.txt
 ```
 
+### ArmNN detector ( orange pi 5 )
+
+You need to put ArmNN binaries for them to be located in a way, when `/usr/lib/ArmNN-linux-aarch64/libarmnnDelegate.so` is correct.
+Download binaries from https://github.com/ARM-software/armnn/releases for your platform
+
+
+```yaml
+detectors:
+  armnn:
+    type: armnn
+    num_threads: 8
+#    model:
+#      path: //cpu_model.tflite  # used by default. 
+
+model:
+  width: 320
+  height: 320
+
+```
+
+In order for GPU to work you need to have `armnn-latest-all` installed as well as `clinfo` should show
+output for the GPU support. See [hardware acceleration](hardware_acceleration.md).
+
 ### Intel NCS2 VPU and Myriad X Setup
 
 Intel produces a neural net inference accelleration chip called Myriad X. This chip was sold in their Neural Compute Stick 2 (NCS2) which has been discontinued. If intending to use the MYRIAD device for accelleration, additional setup is required to pass through the USB device. The host needs a udev rule installed to handle the NCS2 device.

docs/docs/configuration/hardware_acceleration.md

@@ -15,6 +15,78 @@ ffmpeg:
   hwaccel_args: preset-rpi-64-h264
 ```
 
+### Orange Pi 5 ( ArmNN )
+
+Ensure you have installed
+
+```sh
+ffmpeg/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,upgradable to: 7:5.1.2-3]
+libavcodec58/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+libavdevice58/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+libavfilter7/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+libavformat58/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+libavutil56/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+libpostproc55/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+librockchip-mpp1/jammy,now 1.5.0-1+git230210.c145c84~jammy1 arm64 [installed,automatic]
+libswresample3/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+libswscale5/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic]
+```
+from https://github.com/orangepi-xunlong/rk-rootfs-build/tree/rk3588_packages_jammy
+
+```yaml
+ffmpeg:
+  hwaccel_args: -hwaccel drm -hwaccel_device /dev/dri/renderD128 -c:v h264_rkmpp
+```
+
+Also, for the CPU and GPU accelleration you should use `armnn` detector on this board [see](detectors.md)
+
+Install packages [see tutorials](https://github.com/ARM-software/armnn/blob/branches/armnn_23_02/InstallationViaAptRepository.md)
+
+```sh
+armnn-latest-all/jammy,now 23.02-1~ubuntu22.04 arm64 [installed]
+armnn-latest-cpu-gpu-ref/jammy,now 23.02-1~ubuntu22.04 arm64 [installed]
+armnn-latest-cpu-gpu/jammy,now 23.02-1~ubuntu22.04 arm64 [installed]
+armnn-latest-cpu/jammy,now 23.02-1~ubuntu22.04 arm64 [installed]
+armnn-latest-gpu/jammy,now 23.02-1~ubuntu22.04 arm64 [installed]
+armnn-latest-ref/jammy,now 23.02-1~ubuntu22.04 arm64 [installed]
+libarmnn-cpuacc-backend32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic]
+libarmnn-cpuref-backend32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic]
+libarmnn-gpuacc-backend32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic]
+libarmnn22/unstable,now 20.08-12 arm64 [installed,automatic]
+libarmnn32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic]
+libarmnnaclcommon22/unstable,now 20.08-12 arm64 [installed]
+libarmnnaclcommon32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic]
+libarmnntfliteparser24/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic]
+```
+
+In order for the GPU to work install packages
+
+```sh
+libmali-g610-x11/jammy,now 1.0.2.4 arm64 [installed]
+libmali-valhall-g610-g6p0-x11-gbm/now 1.9-1 arm64 [installed,local]
+```
+
+for Ubuntu
+
+```sh
+apt install ocl-icd-opencl-dev
+mkdir -p /etc/OpenCL/vendors/
+dpkg -i libmali-valhall-g610-g6p0-x11_1.9-1_arm64.deb
+```
+
+`clinfo | grep 'Device Name'` should show you a full output of available data about Mali GPU
+
+```sh
+root@23cfa5ff7203:/opt/frigate# clinfo | grep 'Device Name'
+  Device Name                                     Mali-LODX r0p0
+    Device Name                                   Mali-LODX r0p0
+    Device Name                                   Mali-LODX r0p0
+    Device Name                                   Mali-LODX r0p0
+```
+
+
+
+
 ### Intel-based CPUs (<10th Generation) via VAAPI
 
 VAAPI supports automatic profile selection so it will work automatically with both H.264 and H.265 streams. VAAPI is recommended for all generations of Intel-based CPUs if QSV does not work.

frigate/detectors/plugins/armnn_tfl.py

@@ -0,0 +1,78 @@
+import logging
+import numpy as np
+
+from frigate.detectors.detection_api import DetectionApi
+from frigate.detectors.detector_config import BaseDetectorConfig
+from typing import Literal
+from pydantic import Extra, Field
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from tensorflow.lite.python.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+DETECTOR_KEY = "armnn"
+
+
+class ArmNNDetectorConfig(BaseDetectorConfig):
+    type: Literal[DETECTOR_KEY]
+    num_threads: int = Field(default=8, title="Number of detection threads")
+
+
+def load_armnn_delegate(library_path, options=None):
+    try:
+        from tflite_runtime.interpreter import load_delegate
+    except ModuleNotFoundError:
+        from tensorflow.lite.python.interpreter import load_delegate
+
+    if options is None:
+        options = {"backends": "CpuAcc,GpuAcc,CpuRef", "logging-severity": "info"}
+
+    return load_delegate(library_path, options=options)
+
+
+class ArmNNTfl(DetectionApi):
+    type_key = DETECTOR_KEY
+
+    def __init__(self, detector_config: ArmNNDetectorConfig):
+        armnn_delegate = load_armnn_delegate("/usr/lib/ArmNN-linux-aarch64/libarmnnDelegate.so")
+
+        self.interpreter = Interpreter(
+            model_path=detector_config.model.path or "/cpu_model.tflite",
+            num_threads=detector_config.num_threads or 8,
+            experimental_delegates=[armnn_delegate],
+        )
+
+        self.interpreter.allocate_tensors()
+
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+
+    def detect_raw(self, tensor_input):
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
+        self.interpreter.invoke()
+
+        boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0]
+        class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0]
+        scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0]
+        count = int(
+            self.interpreter.tensor(self.tensor_output_details[3]["index"])()[0]
+        )
+
+        detections = np.zeros((20, 6), np.float32)
+
+        for i in range(count):
+            if scores[i] < 0.4 or i == 20:
+                break
+            detections[i] = [
+                class_ids[i],
+                float(scores[i]),
+                boxes[i][0],
+                boxes[i][1],
+                boxes[i][2],
+                boxes[i][3],
+            ]
+
+        return detections