LibWeb+LibGfx: Use GPU backend for <canvas>

This is implemented by using a GPU-accelerated surface for <canvas> when
a GPU context is available.

A side effect of this change is that all canvas modifications have to be
performed through Gfx::Painter, and whenever its content has to be
accessed, we need to take a snapshot of the corresponding GPU surface.

A new DrawPaintingSurface display list command is introduced to allow
cheap blitting of canvas content without having to read GPU surface
content into RAM.

Userland/Libraries/LibGfx/Painter.cpp

@@ -6,14 +6,16 @@
 
 #include <LibGfx/Painter.h>
 #include <LibGfx/PainterSkia.h>
+#include <LibGfx/PaintingSurface.h>
 
 namespace Gfx {
 
 Painter::~Painter() = default;
 
 NonnullOwnPtr<Painter> Painter::create(NonnullRefPtr<Gfx::Bitmap> target_bitmap)
 {
-    return make<PainterSkia>(move(target_bitmap));
+    auto painting_surface = PaintingSurface::wrap_bitmap(target_bitmap);
+    return make<PainterSkia>(painting_surface);
 }
 
 }

Userland/Libraries/LibGfx/PainterSkia.cpp

@@ -56,21 +56,17 @@ static SkAlphaType to_skia_alpha_type(Gfx::AlphaType alpha_type)
 }
 
 struct PainterSkia::Impl {
-    NonnullRefPtr<Gfx::Bitmap> gfx_bitmap;
-    OwnPtr<SkBitmap> sk_bitmap;
-    OwnPtr<SkCanvas> sk_canvas;
+    RefPtr<Gfx::PaintingSurface> painting_surface;
 
-    Impl(NonnullRefPtr<Gfx::Bitmap> target_bitmap)
-        : gfx_bitmap(move(target_bitmap))
+    Impl(Gfx::PaintingSurface& surface)
+        : painting_surface(surface)
     {
-        sk_bitmap = make<SkBitmap>();
-        SkImageInfo info = SkImageInfo::Make(gfx_bitmap->width(), gfx_bitmap->height(), to_skia_color_type(gfx_bitmap->format()), to_skia_alpha_type(gfx_bitmap->alpha_type()));
-        sk_bitmap->installPixels(info, gfx_bitmap->scanline(0), gfx_bitmap->pitch());
-
-        sk_canvas = make<SkCanvas>(*sk_bitmap);
     }
 
-    SkCanvas* canvas() { return sk_canvas; }
+    SkCanvas* canvas() const
+    {
+        return &painting_surface->canvas();
+    }
 };
 
 static constexpr SkRect to_skia_rect(auto const& rect)
@@ -99,8 +95,8 @@ static SkPathFillType to_skia_path_fill_type(Gfx::WindingRule winding_rule)
     VERIFY_NOT_REACHED();
 }
 
-PainterSkia::PainterSkia(NonnullRefPtr<Gfx::Bitmap> target_bitmap)
-    : m_impl(adopt_own(*new Impl { move(target_bitmap) }))
+PainterSkia::PainterSkia(NonnullRefPtr<Gfx::PaintingSurface> painting_surface)
+    : m_impl(adopt_own(*new Impl { move(painting_surface) }))
 {
 }
 

Userland/Libraries/LibGfx/PainterSkia.h

@@ -9,12 +9,13 @@
 #include <AK/NonnullOwnPtr.h>
 #include <LibGfx/Bitmap.h>
 #include <LibGfx/Painter.h>
+#include <LibGfx/PaintingSurface.h>
 
 namespace Gfx {
 
 class PainterSkia final : public Painter {
 public:
-    explicit PainterSkia(NonnullRefPtr<Gfx::Bitmap>);
+    explicit PainterSkia(NonnullRefPtr<Gfx::PaintingSurface>);
     virtual ~PainterSkia() override;
 
     virtual void clear_rect(Gfx::FloatRect const&, Color) override;

Userland/Libraries/LibWeb/HTML/Canvas/CanvasDrawImage.cpp

@@ -11,15 +11,25 @@ namespace Web::HTML {
 
 static void default_source_size(CanvasImageSource const& image, float& source_width, float& source_height)
 {
-    image.visit([&source_width, &source_height](auto const& source) {
-        if (source->bitmap()) {
-            source_width = source->bitmap()->width();
-            source_height = source->bitmap()->height();
-        } else {
-            source_width = source->width();
-            source_height = source->height();
-        }
-    });
+    image.visit(
+        [&source_width, &source_height](JS::Handle<HTMLCanvasElement> const& source) {
+            if (source->surface()) {
+                source_width = source->surface()->size().width();
+                source_height = source->surface()->size().height();
+            } else {
+                source_width = source->width();
+                source_height = source->height();
+            }
+        },
+        [&source_width, &source_height](auto const& source) {
+            if (source->bitmap()) {
+                source_width = source->bitmap()->width();
+                source_height = source->bitmap()->height();
+            } else {
+                source_width = source->width();
+                source_height = source->height();
+            }
+        });
 }
 
 WebIDL::ExceptionOr<void> CanvasDrawImage::draw_image(Web::HTML::CanvasImageSource const& image, float destination_x, float destination_y)

Userland/Libraries/LibWeb/HTML/CanvasPattern.cpp

@@ -126,10 +126,14 @@ WebIDL::ExceptionOr<JS::GCPtr<CanvasPattern>> CanvasPattern::create(JS::Realm& r
         return WebIDL::SyntaxError::create(realm, "Repetition value is not valid"_fly_string);
 
     // Note: Bitmap won't be null here, as if it were it would have "bad" usability.
-    auto const& bitmap = *image.visit([](auto const& source) -> Gfx::Bitmap const* { return source->bitmap(); });
+    auto bitmap = image.visit(
+        [](JS::Handle<HTMLCanvasElement> const& source) -> RefPtr<Gfx::Bitmap> { return source->surface()->create_snapshot(); },
+        [](JS::Handle<HTMLImageElement> const& source) -> RefPtr<Gfx::Bitmap> { return *source->bitmap(); },
+        [](JS::Handle<ImageBitmap> const& source) -> RefPtr<Gfx::Bitmap> { return *source->bitmap(); },
+        [](JS::Handle<ImageData> const& source) -> RefPtr<Gfx::Bitmap> { return &source->bitmap(); });
 
     // 6. Let pattern be a new CanvasPattern object with the image image and the repetition behavior given by repetition.
-    auto pattern = TRY_OR_THROW_OOM(realm.vm(), CanvasPatternPaintStyle::create(bitmap, *repetition_value));
+    auto pattern = TRY_OR_THROW_OOM(realm.vm(), CanvasPatternPaintStyle::create(*bitmap, *repetition_value));
 
     // FIXME: 7. If image is not origin-clean, then mark pattern as not origin-clean.
 

Userland/Libraries/LibWeb/HTML/CanvasRenderingContext2D.cpp

@@ -8,6 +8,7 @@
 
 #include <AK/OwnPtr.h>
 #include <LibGfx/DeprecatedPainter.h>
+#include <LibGfx/PainterSkia.h>
 #include <LibGfx/Quad.h>
 #include <LibGfx/Rect.h>
 #include <LibUnicode/Segmenter.h>
@@ -120,7 +121,11 @@ WebIDL::ExceptionOr<void> CanvasRenderingContext2D::draw_image_internal(CanvasIm
     if (usability == CanvasImageSourceUsability::Bad)
         return {};
 
-    auto const* bitmap = image.visit([](auto const& source) -> Gfx::Bitmap const* { return source->bitmap(); });
+    auto const bitmap = image.visit(
+        [](JS::Handle<HTMLCanvasElement> const& source) -> RefPtr<Gfx::Bitmap const> { return source->surface()->create_snapshot(); },
+        [](JS::Handle<HTMLImageElement> const& source) -> RefPtr<Gfx::Bitmap const> { return source->bitmap(); },
+        [](JS::Handle<ImageBitmap> const& source) -> RefPtr<Gfx::Bitmap const> { return source->bitmap(); },
+        [](JS::Handle<ImageData> const& source) -> RefPtr<Gfx::Bitmap const> { return source->bitmap(); });
     if (!bitmap)
         return {};
 
@@ -179,11 +184,11 @@ void CanvasRenderingContext2D::did_draw(Gfx::FloatRect const&)
 
 Gfx::Painter* CanvasRenderingContext2D::painter()
 {
-    if (!canvas_element().bitmap()) {
-        if (!canvas_element().create_bitmap())
+    if (!canvas_element().surface()) {
+        if (!canvas_element().allocate_painting_surface())
             return nullptr;
         canvas_element().document().invalidate_display_list();
-        m_painter = Gfx::Painter::create(*canvas_element().bitmap());
+        m_painter = make<Gfx::PainterSkia>(*canvas_element().surface());
     }
     return m_painter.ptr();
 }
@@ -341,23 +346,23 @@ WebIDL::ExceptionOr<JS::GCPtr<ImageData>> CanvasRenderingContext2D::get_image_da
     auto image_data = TRY(ImageData::create(realm(), width, height, settings));
 
     // NOTE: We don't attempt to create the underlying bitmap here; if it doesn't exist, it's like copying only transparent black pixels (which is a no-op).
-    if (!canvas_element().bitmap())
+    if (!canvas_element().surface())
         return image_data;
-    auto const& bitmap = *canvas_element().bitmap();
+    auto const bitmap = canvas_element().surface()->create_snapshot();
 
     // 5. Let the source rectangle be the rectangle whose corners are the four points (sx, sy), (sx+sw, sy), (sx+sw, sy+sh), (sx, sy+sh).
     auto source_rect = Gfx::Rect { x, y, width, height };
-    auto source_rect_intersected = source_rect.intersected(bitmap.rect());
+    auto source_rect_intersected = source_rect.intersected(bitmap->rect());
 
     // 6. Set the pixel values of imageData to be the pixels of this's output bitmap in the area specified by the source rectangle in the bitmap's coordinate space units, converted from this's color space to imageData's colorSpace using 'relative-colorimetric' rendering intent.
     // NOTE: Internally we must use premultiplied alpha, but ImageData should hold unpremultiplied alpha. This conversion
     //       might result in a loss of precision, but is according to spec.
     //       See: https://html.spec.whatwg.org/multipage/canvas.html#premultiplied-alpha-and-the-2d-rendering-context
-    ASSERT(bitmap.alpha_type() == Gfx::AlphaType::Premultiplied);
+    ASSERT(bitmap->alpha_type() == Gfx::AlphaType::Premultiplied);
     ASSERT(image_data->bitmap().alpha_type() == Gfx::AlphaType::Unpremultiplied);
 
     auto painter = Gfx::Painter::create(image_data->bitmap());
-    painter->draw_bitmap(image_data->bitmap().rect().to_type<float>(), bitmap, source_rect_intersected, Gfx::ScalingMode::NearestNeighbor, drawing_state().global_alpha);
+    painter->draw_bitmap(image_data->bitmap().rect().to_type<float>(), *bitmap, source_rect_intersected, Gfx::ScalingMode::NearestNeighbor, drawing_state().global_alpha);
 
     // 7. Set the pixels values of imageData for areas of the source rectangle that are outside of the output bitmap to transparent black.
     // NOTE: No-op, already done during creation.
@@ -378,11 +383,11 @@ void CanvasRenderingContext2D::put_image_data(ImageData const& image_data, float
 // https://html.spec.whatwg.org/multipage/canvas.html#reset-the-rendering-context-to-its-default-state
 void CanvasRenderingContext2D::reset_to_default_state()
 {
-    auto* bitmap = canvas_element().bitmap();
+    auto surface = canvas_element().surface();
 
     // 1. Clear canvas's bitmap to transparent black.
-    if (bitmap) {
-        painter()->clear_rect(bitmap->rect().to_type<float>(), Color::Transparent);
+    if (surface) {
+        painter()->clear_rect(surface->rect().to_type<float>(), Color::Transparent);
     }
 
     // 2. Empty the list of subpaths in context's current default path.
@@ -394,8 +399,8 @@ void CanvasRenderingContext2D::reset_to_default_state()
     // 4. Reset everything that drawing state consists of to their initial values.
     reset_drawing_state();
 
-    if (bitmap)
-        did_draw(bitmap->rect().to_type<float>());
+    if (surface)
+        did_draw(surface->rect().to_type<float>());
 }
 
 // https://html.spec.whatwg.org/multipage/canvas.html#dom-context-2d-measuretext

Userland/Libraries/LibWeb/HTML/HTMLCanvasElement.cpp

@@ -21,6 +21,7 @@
 #include <LibWeb/HTML/HTMLCanvasElement.h>
 #include <LibWeb/HTML/Numbers.h>
 #include <LibWeb/HTML/Scripting/ExceptionReporter.h>
+#include <LibWeb/HTML/TraversableNavigable.h>
 #include <LibWeb/Layout/CanvasBox.h>
 #include <LibWeb/Platform/EventLoopPlugin.h>
 #include <LibWeb/WebIDL/AbstractOperations.h>
@@ -115,15 +116,15 @@ void HTMLCanvasElement::reset_context_to_default_state()
 WebIDL::ExceptionOr<void> HTMLCanvasElement::set_width(unsigned value)
 {
     TRY(set_attribute(HTML::AttributeNames::width, MUST(String::number(value))));
-    m_bitmap = nullptr;
+    m_surface = nullptr;
     reset_context_to_default_state();
     return {};
 }
 
 WebIDL::ExceptionOr<void> HTMLCanvasElement::set_height(unsigned value)
 {
     TRY(set_attribute(HTML::AttributeNames::height, MUST(String::number(value))));
-    m_bitmap = nullptr;
+    m_surface = nullptr;
     reset_context_to_default_state();
     return {};
 }
@@ -204,20 +205,23 @@ static Gfx::IntSize bitmap_size_for_canvas(HTMLCanvasElement const& canvas, size
     return Gfx::IntSize(width, height);
 }
 
-bool HTMLCanvasElement::create_bitmap(size_t minimum_width, size_t minimum_height)
+bool HTMLCanvasElement::allocate_painting_surface(size_t minimum_width, size_t minimum_height)
 {
+    if (m_surface)
+        return true;
+
+    auto traversable = document().navigable()->traversable_navigable();
+    VERIFY(traversable);
+
     auto size = bitmap_size_for_canvas(*this, minimum_width, minimum_height);
     if (size.is_empty()) {
-        m_bitmap = nullptr;
+        m_surface = nullptr;
         return false;
     }
-    if (!m_bitmap || m_bitmap->size() != size) {
-        auto bitmap_or_error = Gfx::Bitmap::create(Gfx::BitmapFormat::BGRA8888, Gfx::AlphaType::Premultiplied, size);
-        if (bitmap_or_error.is_error())
-            return false;
-        m_bitmap = bitmap_or_error.release_value_but_fixme_should_propagate_errors();
+    if (!m_surface || m_surface->size() != size) {
+        m_surface = Gfx::PaintingSurface::create_with_size(traversable->skia_backend_context(), size, Gfx::BitmapFormat::BGRA8888, Gfx::AlphaType::Premultiplied);
     }
-    return m_bitmap;
+    return m_surface;
 }
 
 struct SerializeBitmapResult {
@@ -252,18 +256,19 @@ static ErrorOr<SerializeBitmapResult> serialize_bitmap(Gfx::Bitmap const& bitmap
 String HTMLCanvasElement::to_data_url(StringView type, Optional<double> quality)
 {
     // It is possible the the canvas doesn't have a associated bitmap so create one
-    if (!bitmap())
-        create_bitmap();
+    if (!m_surface) {
+        allocate_painting_surface();
+    }
 
     // FIXME: 1. If this canvas element's bitmap's origin-clean flag is set to false, then throw a "SecurityError" DOMException.
 
     // 2. If this canvas element's bitmap has no pixels (i.e. either its horizontal dimension or its vertical dimension is zero)
     //    then return the string "data:,". (This is the shortest data: URL; it represents the empty string in a text/plain resource.)
-    if (!m_bitmap)
+    if (!m_surface)
         return "data:,"_string;
 
     // 3. Let file be a serialization of this canvas element's bitmap as a file, passing type and quality if given.
-    auto file = serialize_bitmap(*m_bitmap, type, move(quality));
+    auto file = serialize_bitmap(*m_surface->create_snapshot(), type, move(quality));
 
     // 4. If file is null then return "data:,".
     if (file.is_error()) {
@@ -283,8 +288,9 @@ String HTMLCanvasElement::to_data_url(StringView type, Optional<double> quality)
 WebIDL::ExceptionOr<void> HTMLCanvasElement::to_blob(JS::NonnullGCPtr<WebIDL::CallbackType> callback, StringView type, Optional<double> quality)
 {
     // It is possible the the canvas doesn't have a associated bitmap so create one
-    if (!bitmap())
-        create_bitmap();
+    if (!m_surface) {
+        allocate_painting_surface();
+    }
 
     // FIXME: 1. If this canvas element's bitmap's origin-clean flag is set to false, then throw a "SecurityError" DOMException.
 
@@ -293,8 +299,9 @@ WebIDL::ExceptionOr<void> HTMLCanvasElement::to_blob(JS::NonnullGCPtr<WebIDL::Ca
 
     // 3. If this canvas element's bitmap has pixels (i.e., neither its horizontal dimension nor its vertical dimension is zero),
     //    then set result to a copy of this canvas element's bitmap.
-    if (m_bitmap)
-        bitmap_result = TRY_OR_THROW_OOM(vm(), m_bitmap->clone());
+    if (m_surface) {
+        bitmap_result = m_surface->create_snapshot();
+    }
 
     // 4. Run these steps in parallel:
     Platform::EventLoopPlugin::the().deferred_invoke([this, callback, bitmap_result, type, quality] {
@@ -326,6 +333,10 @@ WebIDL::ExceptionOr<void> HTMLCanvasElement::to_blob(JS::NonnullGCPtr<WebIDL::Ca
 
 void HTMLCanvasElement::present()
 {
+    if (m_surface) {
+        m_surface->flush();
+    }
+
     m_context.visit(
         [](JS::NonnullGCPtr<CanvasRenderingContext2D>&) {
             // Do nothing, CRC2D writes directly to the canvas bitmap.

Userland/Libraries/LibWeb/HTML/HTMLCanvasElement.h

@@ -8,6 +8,7 @@
 
 #include <AK/ByteBuffer.h>
 #include <LibGfx/Forward.h>
+#include <LibGfx/PaintingSurface.h>
 #include <LibWeb/HTML/HTMLElement.h>
 #include <LibWeb/WebGL/WebGLRenderingContext.h>
 
@@ -22,9 +23,9 @@ class HTMLCanvasElement final : public HTMLElement {
 
     virtual ~HTMLCanvasElement() override;
 
-    Gfx::Bitmap const* bitmap() const { return m_bitmap; }
-    Gfx::Bitmap* bitmap() { return m_bitmap; }
-    bool create_bitmap(size_t minimum_width = 0, size_t minimum_height = 0);
+    bool allocate_painting_surface(size_t minimum_width = 0, size_t minimum_height = 0);
+    RefPtr<Gfx::PaintingSurface> surface() { return m_surface; }
+    RefPtr<Gfx::PaintingSurface const> surface() const { return m_surface; }
 
     JS::ThrowCompletionOr<RenderingContext> get_context(String const& type, JS::Value options);
 
@@ -58,7 +59,7 @@ class HTMLCanvasElement final : public HTMLElement {
     JS::ThrowCompletionOr<HasOrCreatedContext> create_webgl_context(JS::Value options);
     void reset_context_to_default_state();
 
-    RefPtr<Gfx::Bitmap> m_bitmap;
+    RefPtr<Gfx::PaintingSurface> m_surface;
 
     Variant<JS::NonnullGCPtr<HTML::CanvasRenderingContext2D>, JS::NonnullGCPtr<WebGL::WebGLRenderingContext>, Empty> m_context;
 };

Userland/Libraries/LibWeb/HTML/TraversableNavigable.h

@@ -97,6 +97,8 @@ class TraversableNavigable final : public Navigable {
 
     void paint(Web::DevicePixelRect const&, Painting::BackingStore&, Web::PaintOptions);
 
+    RefPtr<Gfx::SkiaBackendContext> skia_backend_context() const { return m_skia_backend_context; }
+
 private:
     TraversableNavigable(JS::NonnullGCPtr<Page>);
 

Userland/Libraries/LibWeb/Painting/CanvasPaintable.cpp

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <LibWeb/HTML/TraversableNavigable.h>
 #include <LibWeb/Painting/CanvasPaintable.h>
 
 namespace Web::Painting {
@@ -36,11 +37,13 @@ void CanvasPaintable::paint(PaintContext& context, PaintPhase phase) const
         auto canvas_rect = context.rounded_device_rect(absolute_rect());
         ScopedCornerRadiusClip corner_clip { context, canvas_rect, normalized_border_radii_data(ShrinkRadiiForBorders::Yes) };
 
-        if (layout_box().dom_node().bitmap()) {
+        if (layout_box().dom_node().surface()) {
+            auto surface = layout_box().dom_node().surface();
+
             // FIXME: Remove this const_cast.
             const_cast<HTML::HTMLCanvasElement&>(layout_box().dom_node()).present();
-            auto scaling_mode = to_gfx_scaling_mode(computed_values().image_rendering(), layout_box().dom_node().bitmap()->rect(), canvas_rect.to_type<int>());
-            context.display_list_recorder().draw_scaled_bitmap(canvas_rect.to_type<int>(), *layout_box().dom_node().bitmap(), layout_box().dom_node().bitmap()->rect(), scaling_mode);
+            auto scaling_mode = to_gfx_scaling_mode(computed_values().image_rendering(), surface->rect(), canvas_rect.to_type<int>());
+            context.display_list_recorder().draw_painting_surface(canvas_rect.to_type<int>(), *layout_box().dom_node().surface(), surface->rect(), scaling_mode);
         }
     }
 }