ImageDecoder+LibGfx: Collate decoded bitmaps before sending over IPC

There is an issue where gifs with many frames cannot be loaded, as each
bitmap is sent over IPC using a separate file descriptor, and there is
limit on the maximum number of descriptors per IPC message. Thus, trying
to load gifs with more than 64 frames (the current limit) causes the
image decoder process to die.

This commit introduces the BitmapSequence class, which is a thin wrapper
around the type Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> and
provides an IPC encode/decode routine that collates all bitmap data into
a single buffer so that only a single file descriptor is required per
IPC transfer, even if multiple frames are being sent.

Meta/gn/secondary/Userland/Libraries/LibGfx/BUILD.gn

@@ -27,6 +27,7 @@ shared_library("LibGfx") {
     "AffineTransform.cpp",
     "AntiAliasingPainter.cpp",
     "Bitmap.cpp",
+    "BitmapSequence.cpp",
     "CMYKBitmap.cpp",
     "Color.cpp",
     "DeltaE.cpp",

Userland/Libraries/LibGfx/BitmapSequence.cpp

@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2024, Zachary Huang <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Forward.h>
+#include <LibCore/AnonymousBuffer.h>
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/BitmapSequence.h>
+#include <LibGfx/Size.h>
+#include <LibIPC/Decoder.h>
+#include <LibIPC/Encoder.h>
+#include <LibIPC/File.h>
+
+namespace Gfx {
+
+static BitmapMetadata get_metadata(Bitmap const& bitmap)
+{
+    return BitmapMetadata { .format = bitmap.format(), .alpha_type = bitmap.alpha_type(), .size = bitmap.size(), .size_in_bytes = bitmap.size_in_bytes() };
+}
+
+}
+
+namespace IPC {
+
+template<>
+ErrorOr<void> encode(Encoder& encoder, Gfx::BitmapMetadata const& metadata)
+{
+    TRY(encoder.encode(static_cast<u32>(metadata.format)));
+    TRY(encoder.encode(static_cast<u32>(metadata.alpha_type)));
+    TRY(encoder.encode(metadata.size_in_bytes));
+    TRY(encoder.encode(metadata.size));
+
+    return {};
+}
+
+template<>
+ErrorOr<Gfx::BitmapMetadata> decode(Decoder& decoder)
+{
+    auto raw_bitmap_format = TRY(decoder.decode<u32>());
+    if (!Gfx::is_valid_bitmap_format(raw_bitmap_format))
+        return Error::from_string_literal("IPC: Invalid Gfx::BitmapSequence format");
+    auto format = static_cast<Gfx::BitmapFormat>(raw_bitmap_format);
+
+    auto raw_alpha_type = TRY(decoder.decode<u32>());
+    if (!Gfx::is_valid_alpha_type(raw_alpha_type))
+        return Error::from_string_literal("IPC: Invalid Gfx::BitmapSequence alpha type");
+    auto alpha_type = static_cast<Gfx::AlphaType>(raw_alpha_type);
+
+    auto size_in_bytes = TRY(decoder.decode<size_t>());
+    auto size = TRY(decoder.decode<Gfx::IntSize>());
+
+    return Gfx::BitmapMetadata { format, alpha_type, size, size_in_bytes };
+}
+
+template<>
+ErrorOr<void> encode(Encoder& encoder, Gfx::BitmapSequence const& bitmap_sequence)
+{
+    auto const& bitmaps = bitmap_sequence.bitmaps;
+
+    Vector<Optional<Gfx::BitmapMetadata>> metadata;
+    metadata.ensure_capacity(bitmaps.size());
+
+    size_t total_buffer_size = 0;
+
+    for (auto const& bitmap_option : bitmaps) {
+        Optional<Gfx::BitmapMetadata> data = {};
+
+        if (bitmap_option.has_value()) {
+            data = get_metadata(bitmap_option.value());
+            total_buffer_size += data->size_in_bytes;
+        }
+
+        metadata.unchecked_append(data);
+    }
+
+    TRY(encoder.encode(metadata));
+
+    // collate all of the bitmap data into one contiguous buffer
+    auto collated_buffer = TRY(Core::AnonymousBuffer::create_with_size(total_buffer_size));
+
+    auto* write_pointer = collated_buffer.data<u8>();
+    for (auto const& bitmap_option : bitmaps) {
+        if (bitmap_option.has_value()) {
+            auto const& bitmap = bitmap_option.value();
+            memcpy(write_pointer, bitmap->scanline(0), bitmap->size_in_bytes());
+            write_pointer += bitmap->size_in_bytes();
+        }
+    }
+
+    TRY(encoder.encode(collated_buffer));
+
+    return {};
+}
+
+template<>
+ErrorOr<Gfx::BitmapSequence> decode(Decoder& decoder)
+{
+    auto metadata = TRY(decoder.decode<Vector<Optional<Gfx::BitmapMetadata>>>());
+    auto collated_buffer = TRY(decoder.decode<Core::AnonymousBuffer>());
+
+    Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+    bitmaps.ensure_capacity(metadata.size());
+
+    ReadonlyBytes bytes = ReadonlyBytes(collated_buffer.data<u8>(), collated_buffer.size());
+    size_t bytes_read = 0;
+
+    // sequentially read each valid bitmap's data from the collated buffer
+    for (auto const& metadata_option : metadata) {
+        Optional<NonnullRefPtr<Gfx::Bitmap>> bitmap = {};
+
+        if (metadata_option.has_value()) {
+            auto metadata = metadata_option.value();
+            size_t size_in_bytes = metadata.size_in_bytes;
+
+            if (bytes_read + size_in_bytes > bytes.size())
+                return Error::from_string_literal("IPC: Invalid Gfx::BitmapSequence buffer data");
+
+            auto buffer = TRY(Core::AnonymousBuffer::create_with_size(size_in_bytes));
+
+            memcpy(buffer.data<u8>(), bytes.slice(bytes_read, size_in_bytes).data(), size_in_bytes);
+            bytes_read += size_in_bytes;
+
+            bitmap = TRY(Gfx::Bitmap::create_with_anonymous_buffer(metadata.format, metadata.alpha_type, move(buffer), metadata.size));
+        }
+
+        bitmaps.append(bitmap);
+    }
+
+    return Gfx::BitmapSequence { bitmaps };
+}
+
+}

Userland/Libraries/LibGfx/BitmapSequence.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2024, Zachary Huang <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/RefPtr.h>
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/Size.h>
+#include <LibIPC/Forward.h>
+
+namespace Gfx {
+
+struct BitmapSequence {
+    Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+};
+
+// a struct to temporarily store bitmap fields before the buffer data is decoded
+struct BitmapMetadata {
+    Gfx::BitmapFormat format;
+    Gfx::AlphaType alpha_type;
+    Gfx::IntSize size;
+    size_t size_in_bytes;
+};
+
+}
+
+namespace IPC {
+
+template<>
+ErrorOr<void> encode(Encoder&, Gfx::BitmapMetadata const&);
+
+template<>
+ErrorOr<Gfx::BitmapMetadata> decode(Decoder&);
+
+template<>
+ErrorOr<void> encode(Encoder&, Gfx::BitmapSequence const&);
+
+template<>
+ErrorOr<Gfx::BitmapSequence> decode(Decoder&);
+
+}

Userland/Libraries/LibGfx/CMakeLists.txt

@@ -4,6 +4,7 @@ set(SOURCES
     AffineTransform.cpp
     AntiAliasingPainter.cpp
     Bitmap.cpp
+    BitmapSequence.cpp
     CMYKBitmap.cpp
     Color.cpp
     DeltaE.cpp

Userland/Libraries/LibImageDecoderClient/Client.cpp

@@ -57,8 +57,9 @@ NonnullRefPtr<Core::Promise<DecodedImage>> Client::decode_image(ReadonlyBytes en
     return promise;
 }
 
-void Client::did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> const& bitmaps, Vector<u32> const& durations, Gfx::FloatPoint scale)
+void Client::did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Gfx::BitmapSequence const& bitmap_sequence, Vector<u32> const& durations, Gfx::FloatPoint scale)
 {
+    auto const& bitmaps = bitmap_sequence.bitmaps;
     VERIFY(!bitmaps.is_empty());
 
     auto maybe_promise = m_pending_decoded_images.take(image_id);

Userland/Libraries/LibImageDecoderClient/Client.h

@@ -41,7 +41,7 @@ class Client final
 private:
     virtual void die() override;
 
-    virtual void did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> const& bitmaps, Vector<u32> const& durations, Gfx::FloatPoint scale) override;
+    virtual void did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Gfx::BitmapSequence const& bitmap_sequence, Vector<u32> const& durations, Gfx::FloatPoint scale) override;
     virtual void did_fail_to_decode_image(i64 image_id, String const& error_message) override;
 
     HashMap<i64, NonnullRefPtr<Core::Promise<DecodedImage>>> m_pending_decoded_images;

Userland/Services/ImageDecoder/ConnectionFromClient.cpp

@@ -104,6 +104,8 @@ static ErrorOr<ConnectionFromClient::DecodeResult> decode_image_to_details(Core:
     result.is_animated = decoder->is_animated();
     result.loop_count = decoder->loop_count();
 
+    Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+
     if (auto maybe_metadata = decoder->metadata(); maybe_metadata.has_value() && is<Gfx::ExifMetadata>(*maybe_metadata)) {
         auto const& exif = static_cast<Gfx::ExifMetadata const&>(maybe_metadata.value());
         if (exif.x_resolution().has_value() && exif.y_resolution().has_value()) {
@@ -116,11 +118,13 @@ static ErrorOr<ConnectionFromClient::DecodeResult> decode_image_to_details(Core:
         }
     }
 
-    decode_image_to_bitmaps_and_durations_with_decoder(*decoder, move(ideal_size), result.bitmaps, result.durations);
+    decode_image_to_bitmaps_and_durations_with_decoder(*decoder, move(ideal_size), bitmaps, result.durations);
 
-    if (result.bitmaps.is_empty())
+    if (bitmaps.is_empty())
         return Error::from_string_literal("Could not decode image");
 
+    result.bitmaps = Gfx::BitmapSequence { bitmaps };
+
     return result;
 }
 

Userland/Services/ImageDecoder/ConnectionFromClient.h

@@ -10,6 +10,7 @@
 #include <ImageDecoder/Forward.h>
 #include <ImageDecoder/ImageDecoderClientEndpoint.h>
 #include <ImageDecoder/ImageDecoderServerEndpoint.h>
+#include <LibGfx/BitmapSequence.h>
 #include <LibIPC/ConnectionFromClient.h>
 #include <LibThreading/BackgroundAction.h>
 
@@ -28,7 +29,7 @@ class ConnectionFromClient final
         bool is_animated = false;
         u32 loop_count = 0;
         Gfx::FloatPoint scale { 1, 1 };
-        Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+        Gfx::BitmapSequence bitmaps;
         Vector<u32> durations;
     };
 

Userland/Services/ImageDecoder/ImageDecoderClient.ipc

@@ -1,7 +1,7 @@
-#include <LibGfx/ShareableBitmap.h>
+#include <LibGfx/BitmapSequence.h>
 
 endpoint ImageDecoderClient
 {
-    did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps, Vector<u32> durations, Gfx::FloatPoint scale) =|
+    did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Gfx::BitmapSequence bitmaps, Vector<u32> durations, Gfx::FloatPoint scale) =|
     did_fail_to_decode_image(i64 image_id, String error_message) =|
 }