From 9249334e47ff31e5d4a62507f1ac45865a0a5ace Mon Sep 17 00:00:00 2001 From: David Plowman Date: Mon, 23 Aug 2021 09:52:18 +0100 Subject: [PATCH] preview: Add a preview window implemented with Qt The Qt preview window is computationally very expensive and should normally be avoided. It does buffer copying, resizing and colour space conversion all using the CPU. However, it does work with X forwarding so may have some application when a remote preview window is absolutely necessary. The Qt preview has to be requested explicitly via the "--qt-preview" option, we never make it automatically. Signed-off-by: David Plowman --- core/options.hpp | 4 + preview/CMakeLists.txt | 15 ++++ preview/preview.cpp | 5 ++ preview/qt_preview.cpp | 183 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 preview/qt_preview.cpp diff --git a/core/options.hpp b/core/options.hpp index 43fe8322..a4034e2e 100644 --- a/core/options.hpp +++ b/core/options.hpp @@ -50,6 +50,8 @@ struct Options "Set the preview window dimensions, given as x,y,width,height e.g. 0,0,640,480") ("fullscreen,f", value(&fullscreen)->default_value(false)->implicit_value(true), "Use a fullscreen preview window") + ("qt-preview", value(&qt_preview)->default_value(false)->implicit_value(true), + "Use Qt-based preview window (WARNING: causes heavy CPU load, fullscreen not supported)") ("hflip", value(&hflip_)->default_value(false)->implicit_value(true), "Request a horizontal flip transform") ("vflip", value(&vflip_)->default_value(false)->implicit_value(true), "Request a vertical flip transform") ("rotation", value(&rotation_)->default_value(0), "Request an image rotation, 0 or 180") @@ -134,6 +136,7 @@ struct Options unsigned int viewfinder_width; unsigned int viewfinder_height; std::string tuning_file; + bool qt_preview; virtual bool Parse(int argc, char *argv[]) { @@ -246,6 +249,7 @@ struct Options else std::cout << " preview: " << preview_x << "," << preview_y << "," << preview_width << "," << preview_height << std::endl; + std::cout << " qt-preview: " << qt_preview << std::endl; std::cout << " transform: " << transformToString(transform) << std::endl; if (roi_width == 0 || roi_height == 0) std::cout << " roi: all" << std::endl; diff --git a/preview/CMakeLists.txt b/preview/CMakeLists.txt index ec53bf2f..147b28e1 100644 --- a/preview/CMakeLists.txt +++ b/preview/CMakeLists.txt @@ -3,6 +3,8 @@ cmake_minimum_required(VERSION 3.6) pkg_check_modules(LIBDRM QUIET libdrm) pkg_check_modules(X11 QUIET x11) pkg_check_modules(EPOXY QUIET epoxy) +pkg_check_modules(QTCORE QUIET Qt5Core) +pkg_check_modules(QTWIDGETS QUIET Qt5Widgets) set(SRC "preview.cpp") set(TARGET_LIBS "") @@ -29,11 +31,24 @@ else() message(WARNING "EGL libraries not found, this display mode will be unavailable!") endif() +set(QT_FOUND 0) +if (QTCORE_FOUND AND QTWIDGETS_FOUND) + message(STATUS "QTCORE_LINK_LIBRARIES=${QTCORE_LINK_LIBRARIES}") + message(STATUS "QTCORE_INCLUDE_DIRS=${QTCORE_INCLUDE_DIRS}") + include_directories(${QTCORE_INCLUDE_DIRS} ${QTWIDGETS_INCLUDE_DIRS}) + set(TARGET_LIBS ${TARGET_LIBS} ${QTCORE_LIBRARIES} ${QTWIDGETS_LIBRARIES}) + set(SRC ${SRC} qt_preview.cpp) + set(QT_FOUND 1) +else() + message(STATUS "QT not found, this display mode will be unavailable") +endif() + add_library(preview null_preview.cpp ${SRC}) target_link_libraries(preview ${TARGET_LIBS}) target_compile_definitions(preview PUBLIC LIBDRM_PRESENT=${DRM_FOUND}) target_compile_definitions(preview PUBLIC LIBEGL_PRESENT=${EGL_FOUND}) +target_compile_definitions(preview PUBLIC QT_PRESENT=${QT_FOUND}) install(TARGETS preview LIBRARY DESTINATION lib ARCHIVE DESTINATION lib) diff --git a/preview/preview.cpp b/preview/preview.cpp index c0f6e7ee..cbc3243f 100644 --- a/preview/preview.cpp +++ b/preview/preview.cpp @@ -10,11 +10,16 @@ Preview *make_null_preview(Options const *options); Preview *make_egl_preview(Options const *options); Preview *make_drm_preview(Options const *options); +Preview *make_qt_preview(Options const *options); Preview *make_preview(Options const *options) { if (options->nopreview) return make_null_preview(options); +#if QT_PRESENT + else if (options->qt_preview) + return make_qt_preview(options); +#endif else { try diff --git a/preview/qt_preview.cpp b/preview/qt_preview.cpp new file mode 100644 index 00000000..295c7894 --- /dev/null +++ b/preview/qt_preview.cpp @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2021, Raspberry Pi (Trading) Ltd. + * + * qt_preview.cpp - Qt preview window + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "preview.hpp" + +class MyMainWindow : public QMainWindow +{ +public: + MyMainWindow() : QMainWindow() {} + bool quit = false; +protected: + void closeEvent(QCloseEvent *event) override + { + event->ignore(); + quit = true; + } +}; + +class MyWidget : public QWidget +{ +public: + MyWidget(QWidget *parent, int w, int h) : QWidget(parent), size(w, h) + { + image = QImage(size, QImage::Format_RGB888); + image.fill(0); + } + QSize size; + QImage image; +protected: + void paintEvent(QPaintEvent *) override + { + QPainter painter(this); + painter.drawImage(rect(), image, image.rect()); + } + QSize sizeHint() const override { return size; } +}; + +class QtPreview : public Preview +{ +public: + QtPreview(Options const *options) : Preview(options) + { + window_width_ = options->preview_width; + window_height_ = options->preview_height; + if (window_width_ % 2 || window_height_ % 2) + throw std::runtime_error("QtPreview: expect even dimensions"); + // This preview window is expensive, so make it small by default. + if (window_width_ == 0 || window_height_ == 0) + window_width_ = 512, window_height_ = 384; + thread_ = std::thread(&QtPreview::threadFunc, this, options); + std::unique_lock lock(mutex_); + while (!pane_) + cond_var_.wait(lock); + if (options->verbose) + std::cout << "Made Qt preview" << std::endl; + } + ~QtPreview() + { + application_->exit(); + thread_.join(); + } + void SetInfoText(const std::string &text) override { main_window_->setWindowTitle(QString::fromStdString(text)); } + virtual void Show(int fd, libcamera::Span span, int width, int height, int stride) override + { + // Cache the x sampling locations for speed. This is a quick nearest neighbour resize. + if (last_image_width_ != width) + { + last_image_width_ = width; + x_locations_.resize(window_width_); + for (int i = 0; i < window_width_; i++) + x_locations_[i] = (i * (width - 1) + (window_width_ - 1) / 2) / (window_width_ - 1); + } + + uint8_t *Y_start = span.data(); + uint8_t *U_start = Y_start + stride * height; + int uv_size = (stride / 2) * (height / 2); + uint8_t *dest = pane_->image.bits(); + + // Possibly this should be locked in case a repaint is happening? In practice the risk + // is only that there might be some tearing, so I don't think we worry. We could speed + // it up by getting the ISP to supply RGB, but I'm not sure I want to handle that extra + // possibility in our main application code, so we'll put up with the slow conversion. + for (int y = 0; y < window_height_; y++) + { + int row = (y * (height - 1) + (window_height_ - 1) / 2) / (window_height_ - 1); + uint8_t *Y_row = Y_start + row * stride; + uint8_t *U_row = U_start + (row / 2) * (stride / 2); + uint8_t *V_row = U_row + uv_size; + for (int x = 0; x < window_width_;) + { + int y_off0 = x_locations_[x++]; + int y_off1 = x_locations_[x++]; + int uv_off0 = y_off0 >> 1; + int uv_off1 = y_off0 >> 1; + int Y0 = Y_row[y_off0]; + int Y1 = Y_row[y_off1]; + int U0 = U_row[uv_off0]; + int V0 = V_row[uv_off0]; + int U1 = U_row[uv_off1]; + int V1 = V_row[uv_off1]; + U0 -= 128; + V0 -= 128; + U1 -= 128; + V1 -= 128; + // What colour space? For the purposes of a preview, we're not too bothered. + int R0 = Y0 + 1.402 * V0; + int G0 = Y0 - 0.345 * U0 - 0.714 * V0; + int B0 = Y0 + 1.771 * U0; + int R1 = Y1 + 1.402 * V1; + int G1 = Y1 - 0.345 * U1 - 0.714 * V1; + int B1 = Y1 + 1.771 * U1; + *(dest++) = std::clamp(R0, 0, 255); + *(dest++) = std::clamp(G0, 0, 255); + *(dest++) = std::clamp(B0, 0, 255); + *(dest++) = std::clamp(R1, 0, 255); + *(dest++) = std::clamp(G1, 0, 255); + *(dest++) = std::clamp(B1, 0, 255); + } + } + + pane_->update(); + + // Return the buffer to the camera system. + done_callback_(fd); + } + // Reset the preview window, clearing the current buffers and being ready to + // show new ones. + void Reset() override {} + // Check if preview window has been shut down. + bool Quit() override { return main_window_->quit; } + +private: + void threadFunc(Options const *options) + { + // This acts as Qt's event loop. Really Qt prefers to own the application's event loop + // but we've supplied our own and only want Qt for rendering. This works, but I + // wouldn't write a proper Qt application like this. + int argc = 0; + char **argv = NULL; + QApplication application(argc, argv); + application_ = &application; + MyMainWindow main_window; + main_window_ = &main_window; + MyWidget pane(&main_window, window_width_, window_height_); + main_window.setCentralWidget(&pane); + // Need to get the window border sizes (it seems to be unreasonably difficult...) + main_window.move(options->preview_x + 2, options->preview_y + 28); + main_window.show(); + pane_ = &pane; + cond_var_.notify_one(); + application.exec(); + } + QApplication *application_ = nullptr; + MyMainWindow *main_window_ = nullptr; + MyWidget *pane_ = nullptr; + std::thread thread_; + std::vector x_locations_; + int last_image_width_ = 0; + int window_width_, window_height_; + std::mutex mutex_; + std::condition_variable cond_var_; +}; + +Preview *make_qt_preview(Options const *options) +{ + return new QtPreview(options); +}