diff --git a/Dockerfile b/Dockerfile index 204fc7c812..1cbe8c757f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -178,6 +178,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility" ENV PATH="/usr/lib/btbn-ffmpeg/bin:/usr/local/go2rtc/bin:/usr/local/nginx/sbin:${PATH}" +# Install mpp empowerred ffmpeg for arm64 +RUN --mount=type=tmpfs,target=/tmp --mount=type=tmpfs,target=/var/cache/apt \ + --mount=type=bind,source=docker/mpp.sh,target=/deps/mpp.sh \ + /deps/mpp.sh + # Install dependencies RUN --mount=type=bind,source=docker/install_deps.sh,target=/deps/install_deps.sh \ /deps/install_deps.sh diff --git a/docker/build_nginx.sh b/docker/build_nginx.sh index fd1432f322..653c1c3565 100755 --- a/docker/build_nginx.sh +++ b/docker/build_nginx.sh @@ -9,11 +9,12 @@ RTMP_MODULE_VERSION="1.2.1" cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list -apt-get update +apt-get -qq update apt-get -yqq build-dep nginx -apt-get -yqq install --no-install-recommends ca-certificates wget +# TODO: move all apt-get installs to dedicated script for having it in a single Docker layer = performance of reruns. +apt-get -yqq install --no-install-recommends ca-certificates wget libaio-dev libpcre3 libpcre3-dev update-ca-certificates -f mkdir /tmp/nginx wget -nv https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz @@ -52,15 +53,21 @@ rm v${RTMP_MODULE_VERSION}.tar.gz cd /tmp/nginx +ARCH=$(uname -m) +CC_OPT="-O3 -Wno-error=implicit-fallthrough" + +if [ "$ARCH" = "aarch64" ]; then + CC_OPT="${CC_OPT} -march=armv8-a+crc" +fi + ./configure --prefix=/usr/local/nginx \ - --with-file-aio \ --with-http_sub_module \ --with-http_ssl_module \ --with-threads \ --add-module=../nginx-vod-module \ --add-module=../nginx-secure-token-module \ --add-module=../nginx-rtmp-module \ - --with-cc-opt="-O3 -Wno-error=implicit-fallthrough" + --with-cc-opt="${CC_OPT}" make -j$(nproc) && make install rm -rf /usr/local/nginx/html /usr/local/nginx/conf/*.default diff --git a/docker/install_deps.sh b/docker/install_deps.sh index 4b6ea7923e..4d20098ba9 100755 --- a/docker/install_deps.sh +++ b/docker/install_deps.sh @@ -4,6 +4,7 @@ set -euxo pipefail apt-get -qq update +# TODO: move all apt-get installs to dedicated script for having it in a single Docker layer = performance of reruns. apt-get -qq install --no-install-recommends -y \ apt-transport-https \ gnupg \ @@ -47,6 +48,13 @@ if [[ "${TARGETARCH}" == "arm" ]]; then fi # ffmpeg -> arm64 +if command -v ffmpeg >/dev/null 2>&1; then + HAS_FFMPEG=1 +else + HAS_FFMPEG=0 +fi + +if [[ "${HAS_FFMPEG}" == 0 ]]; then if [[ "${TARGETARCH}" == "arm64" ]]; then # add raspberry pi repo gpg --no-default-keyring --keyring /usr/share/keyrings/raspbian.gpg --keyserver keyserver.ubuntu.com --recv-keys 82B129927FA3303E @@ -54,6 +62,8 @@ if [[ "${TARGETARCH}" == "arm64" ]]; then apt-get -qq update apt-get -qq install --no-install-recommends --no-install-suggests -y ffmpeg fi +fi + # arch specific packages if [[ "${TARGETARCH}" == "amd64" ]]; then diff --git a/docker/mpp.sh b/docker/mpp.sh new file mode 100755 index 0000000000..6b08e3ea9a --- /dev/null +++ b/docker/mpp.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -euxo pipefail + +apt-get -qq update +apt install -y pkg-config + +if [ -e /usr/local/include/mpp/mpp.h ] || [ -e /usr/include/mpp/mpp.h ] || pkg-config --exists rockchip-mpp; then + HAS_MPP=1 +else +# Install mpp from sources +apt-get -qq update +# TODO: move all apt-get installs to dedicated script for having it in a single Docker layer = performance of reruns. +apt-get install -y git build-essential yasm pkg-config \ +libtool coreutils autoconf automake build-essential cmake \ +doxygen git graphviz imagemagick libasound2-dev libass-dev \ +libavcodec-dev libavdevice-dev libavfilter-dev libavformat-dev \ +libavutil-dev libfreetype6-dev libgmp-dev libmp3lame-dev \ +libopencore-amrnb-dev libopencore-amrwb-dev libopus-dev \ +librtmp-dev libsdl2-dev libsdl2-image-dev libsdl2-mixer-dev \ +libsdl2-net-dev libsdl2-ttf-dev libsnappy-dev libsoxr-dev \ +libssh-dev libssl-dev libtool libv4l-dev libva-dev libvdpau-dev \ +libvo-amrwbenc-dev libvorbis-dev libwebp-dev libx264-dev libx265-dev \ +libxcb-shape0-dev libxcb-shm0-dev libxcb-xfixes0-dev libxcb1-dev \ +libxml2-dev lzma-dev meson nasm pkg-config python3-dev \ +python3-pip texinfo wget yasm zlib1g-dev libdrm-dev libaom-dev libdav1d-dev \ +libmp3lame-dev + +cd /tmp +git clone https://github.com/rockchip-linux/mpp.git +cd mpp +mkdir build || true && cd build + +ARCH=$(uname -m) +EXTRA_CFLAGS="" +EXTRA_CXXFLAGS="" + +if [ "$ARCH" = "aarch64" ]; then + EXTRA_CFLAGS="-march=armv8-a+crc" + EXTRA_CXXFLAGS="-march=armv8-a+crc" +fi + +cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_C_FLAGS="${EXTRA_CFLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_CXXFLAGS}" ../ +make -j$(nproc) +make install +ldconfig + +fi + +# TODO: consider moving ffmpeg compillation to a dedicated ffmpeg.sh script +if command -v ffmpeg >/dev/null 2>&1; then + HAS_FFMPEG=1 +else + HAS_FFMPEG=0 +fi + +if [[ "${HAS_FFMPEG}" == 1 ]]; then +# To avoid possible race condition. + apt -y remove ffmpeg +fi +# Compile ffmpeg + +cd /tmp +git clone https://github.com/FFmpeg/FFmpeg.git +cd FFmpeg + +ARCH=$(uname -m) +EXTRA_CFLAGS="-I/usr/local/include" +EXTRA_LDFLAGS="-L/usr/local/lib" + +if [ "$ARCH" = "aarch64" ]; then + EXTRA_CFLAGS="${EXTRA_CFLAGS} -march=armv8-a+crc" +fi + +PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" ./configure \ + --enable-rkmpp \ + --extra-cflags="${EXTRA_CFLAGS}" \ + --extra-ldflags="${EXTRA_LDFLAGS}" \ + --extra-libs="-lpthread -lm -latomic" \ + --arch=arm64 \ + --enable-gmp \ + --enable-gpl \ + --enable-libaom \ + --enable-libass \ + --enable-libdav1d \ + --enable-libdrm \ + --enable-libfreetype \ + --enable-libmp3lame \ + --enable-libopencore-amrnb \ + --enable-libopencore-amrwb \ + --enable-libopus \ + --enable-librtmp \ + --enable-libsnappy \ + --enable-libsoxr \ + --enable-libssh \ + --enable-libvorbis \ + --enable-libwebp \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libxml2 \ + --enable-nonfree \ + --enable-version3 \ + --target-os=linux \ + --enable-pthreads \ + --enable-openssl \ + --enable-hardcoded-tables + +make -j$(nproc) +make install +ldconfig + +cd /tmp +rm -rf mpp +rm -rf FFmpeg diff --git a/docs/docs/configuration/detectors.md b/docs/docs/configuration/detectors.md index b7f442c313..07ba585fc3 100644 --- a/docs/docs/configuration/detectors.md +++ b/docs/docs/configuration/detectors.md @@ -138,6 +138,29 @@ model: labelmap_path: /path/to/coco_80cl.txt ``` +### ArmNN detector ( orange pi 5 ) + +You need to put ArmNN binaries for them to be located in a way, when `/usr/lib/ArmNN-linux-aarch64/libarmnnDelegate.so` is correct. +Download binaries from https://github.com/ARM-software/armnn/releases for your platform + + +```yaml +detectors: + armnn: + type: armnn + num_threads: 8 +# model: +# path: //cpu_model.tflite # used by default. + +model: + width: 320 + height: 320 + +``` + +In order for GPU to work you need to have `armnn-latest-all` installed as well as `clinfo` should show +output for the GPU support. See [hardware acceleration](hardware_acceleration.md). + ### Intel NCS2 VPU and Myriad X Setup Intel produces a neural net inference accelleration chip called Myriad X. This chip was sold in their Neural Compute Stick 2 (NCS2) which has been discontinued. If intending to use the MYRIAD device for accelleration, additional setup is required to pass through the USB device. The host needs a udev rule installed to handle the NCS2 device. diff --git a/docs/docs/configuration/hardware_acceleration.md b/docs/docs/configuration/hardware_acceleration.md index c01bf74ebf..e4012e5493 100644 --- a/docs/docs/configuration/hardware_acceleration.md +++ b/docs/docs/configuration/hardware_acceleration.md @@ -15,6 +15,78 @@ ffmpeg: hwaccel_args: preset-rpi-64-h264 ``` +### Orange Pi 5 ( ArmNN ) + +Ensure you have installed + +```sh +ffmpeg/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,upgradable to: 7:5.1.2-3] +libavcodec58/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +libavdevice58/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +libavfilter7/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +libavformat58/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +libavutil56/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +libpostproc55/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +librockchip-mpp1/jammy,now 1.5.0-1+git230210.c145c84~jammy1 arm64 [installed,automatic] +libswresample3/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +libswscale5/jammy,now 7:4.4.2-0ubuntu0.22.04.1+rkmpp20230207 arm64 [installed,automatic] +``` +from https://github.com/orangepi-xunlong/rk-rootfs-build/tree/rk3588_packages_jammy + +```yaml +ffmpeg: + hwaccel_args: -hwaccel drm -hwaccel_device /dev/dri/renderD128 -c:v h264_rkmpp +``` + +Also, for the CPU and GPU accelleration you should use `armnn` detector on this board [see](detectors.md) + +Install packages [see tutorials](https://github.com/ARM-software/armnn/blob/branches/armnn_23_02/InstallationViaAptRepository.md) + +```sh +armnn-latest-all/jammy,now 23.02-1~ubuntu22.04 arm64 [installed] +armnn-latest-cpu-gpu-ref/jammy,now 23.02-1~ubuntu22.04 arm64 [installed] +armnn-latest-cpu-gpu/jammy,now 23.02-1~ubuntu22.04 arm64 [installed] +armnn-latest-cpu/jammy,now 23.02-1~ubuntu22.04 arm64 [installed] +armnn-latest-gpu/jammy,now 23.02-1~ubuntu22.04 arm64 [installed] +armnn-latest-ref/jammy,now 23.02-1~ubuntu22.04 arm64 [installed] +libarmnn-cpuacc-backend32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic] +libarmnn-cpuref-backend32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic] +libarmnn-gpuacc-backend32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic] +libarmnn22/unstable,now 20.08-12 arm64 [installed,automatic] +libarmnn32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic] +libarmnnaclcommon22/unstable,now 20.08-12 arm64 [installed] +libarmnnaclcommon32/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic] +libarmnntfliteparser24/jammy,now 23.02-1~ubuntu22.04 arm64 [installed,automatic] +``` + +In order for the GPU to work install packages + +```sh +libmali-g610-x11/jammy,now 1.0.2.4 arm64 [installed] +libmali-valhall-g610-g6p0-x11-gbm/now 1.9-1 arm64 [installed,local] +``` + +for Ubuntu + +```sh +apt install ocl-icd-opencl-dev +mkdir -p /etc/OpenCL/vendors/ +dpkg -i libmali-valhall-g610-g6p0-x11_1.9-1_arm64.deb +``` + +`clinfo | grep 'Device Name'` should show you a full output of available data about Mali GPU + +```sh +root@23cfa5ff7203:/opt/frigate# clinfo | grep 'Device Name' + Device Name Mali-LODX r0p0 + Device Name Mali-LODX r0p0 + Device Name Mali-LODX r0p0 + Device Name Mali-LODX r0p0 +``` + + + + ### Intel-based CPUs (<10th Generation) via VAAPI VAAPI supports automatic profile selection so it will work automatically with both H.264 and H.265 streams. VAAPI is recommended for all generations of Intel-based CPUs if QSV does not work. diff --git a/frigate/detectors/plugins/armnn_tfl.py b/frigate/detectors/plugins/armnn_tfl.py new file mode 100644 index 0000000000..20d00f3c57 --- /dev/null +++ b/frigate/detectors/plugins/armnn_tfl.py @@ -0,0 +1,78 @@ +import logging +import numpy as np + +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig +from typing import Literal +from pydantic import Extra, Field + +try: + from tflite_runtime.interpreter import Interpreter +except ModuleNotFoundError: + from tensorflow.lite.python.interpreter import Interpreter + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "armnn" + + +class ArmNNDetectorConfig(BaseDetectorConfig): + type: Literal[DETECTOR_KEY] + num_threads: int = Field(default=8, title="Number of detection threads") + + +def load_armnn_delegate(library_path, options=None): + try: + from tflite_runtime.interpreter import load_delegate + except ModuleNotFoundError: + from tensorflow.lite.python.interpreter import load_delegate + + if options is None: + options = {"backends": "CpuAcc,GpuAcc,CpuRef", "logging-severity": "info"} + + return load_delegate(library_path, options=options) + + +class ArmNNTfl(DetectionApi): + type_key = DETECTOR_KEY + + def __init__(self, detector_config: ArmNNDetectorConfig): + armnn_delegate = load_armnn_delegate("/usr/lib/ArmNN-linux-aarch64/libarmnnDelegate.so") + + self.interpreter = Interpreter( + model_path=detector_config.model.path or "/cpu_model.tflite", + num_threads=detector_config.num_threads or 8, + experimental_delegates=[armnn_delegate], + ) + + self.interpreter.allocate_tensors() + + self.tensor_input_details = self.interpreter.get_input_details() + self.tensor_output_details = self.interpreter.get_output_details() + + def detect_raw(self, tensor_input): + self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) + self.interpreter.invoke() + + boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] + class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] + scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] + count = int( + self.interpreter.tensor(self.tensor_output_details[3]["index"])()[0] + ) + + detections = np.zeros((20, 6), np.float32) + + for i in range(count): + if scores[i] < 0.4 or i == 20: + break + detections[i] = [ + class_ids[i], + float(scores[i]), + boxes[i][0], + boxes[i][1], + boxes[i][2], + boxes[i][3], + ] + + return detections