ImageDecoder+LibGfx: Collate decoded bitmaps before sending over IPC

There is an issue where gifs with many frames cannot be loaded, as each
bitmap is sent over IPC using a separate file descriptor, and there is
limit on the maximum number of descriptors per IPC message. Thus, trying
to load gifs with more than 64 frames (the current limit) causes the
image decoder process to die.

This commit introduces the BitmapSequence class, which is a thin wrapper
around the type Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> and
provides an IPC encode/decode routine that collates all bitmap data into
a single buffer so that only a single file descriptor is required per
IPC transfer, even if multiple frames are being sent.

(cherry picked from commit e0bd42be9590b967d0b5788ce7537a861a35ba74;
amended to fix conflict on build files since we still BitmapMixer.cpp
that upstream removed in LadybirdBrowser/ladybird#41, and
BitmapSequence.cpp conflicted with that. Also amended to not have
AlphaType in BitmapMetadata since we don't have that, and to have
scale instead, which we still do have. Also updated encode() and
decode() to send things in the order they're in in the struct.)

Meta/gn/secondary/Userland/Libraries/LibGfx/BUILD.gn

@@ -30,6 +30,7 @@ shared_library("LibGfx") {
     "AntiAliasingPainter.cpp",
     "Bitmap.cpp",
     "BitmapMixer.cpp",
+    "BitmapSequence.cpp",
     "CMYKBitmap.cpp",
     "ClassicStylePainter.cpp",
     "ClassicWindowTheme.cpp",

Userland/Libraries/LibGfx/BitmapSequence.cpp

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2024, Zachary Huang <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Forward.h>
+#include <LibCore/AnonymousBuffer.h>
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/BitmapSequence.h>
+#include <LibGfx/Size.h>
+#include <LibIPC/Decoder.h>
+#include <LibIPC/Encoder.h>
+#include <LibIPC/File.h>
+
+namespace Gfx {
+
+static BitmapMetadata get_metadata(Bitmap const& bitmap)
+{
+    return BitmapMetadata { .format = bitmap.format(), .size = bitmap.size(), .scale = bitmap.scale(), .size_in_bytes = bitmap.size_in_bytes() };
+}
+
+}
+
+namespace IPC {
+
+template<>
+ErrorOr<void> encode(Encoder& encoder, Gfx::BitmapMetadata const& metadata)
+{
+    TRY(encoder.encode(static_cast<u32>(metadata.format)));
+    TRY(encoder.encode(metadata.size));
+    TRY(encoder.encode(metadata.scale));
+    TRY(encoder.encode(metadata.size_in_bytes));
+
+    return {};
+}
+
+template<>
+ErrorOr<Gfx::BitmapMetadata> decode(Decoder& decoder)
+{
+    auto raw_bitmap_format = TRY(decoder.decode<u32>());
+    if (!Gfx::is_valid_bitmap_format(raw_bitmap_format))
+        return Error::from_string_literal("IPC: Invalid Gfx::BitmapSequence format");
+    auto format = static_cast<Gfx::BitmapFormat>(raw_bitmap_format);
+
+    auto size = TRY(decoder.decode<Gfx::IntSize>());
+    auto scale = TRY(decoder.decode<int>());
+    auto size_in_bytes = TRY(decoder.decode<size_t>());
+
+    return Gfx::BitmapMetadata { format, size, scale, size_in_bytes };
+}
+
+template<>
+ErrorOr<void> encode(Encoder& encoder, Gfx::BitmapSequence const& bitmap_sequence)
+{
+    auto const& bitmaps = bitmap_sequence.bitmaps;
+
+    Vector<Optional<Gfx::BitmapMetadata>> metadata;
+    metadata.ensure_capacity(bitmaps.size());
+
+    size_t total_buffer_size = 0;
+
+    for (auto const& bitmap_option : bitmaps) {
+        Optional<Gfx::BitmapMetadata> data = {};
+
+        if (bitmap_option.has_value()) {
+            data = get_metadata(bitmap_option.value());
+            total_buffer_size += data->size_in_bytes;
+        }
+
+        metadata.unchecked_append(data);
+    }
+
+    TRY(encoder.encode(metadata));
+
+    // collate all of the bitmap data into one contiguous buffer
+    auto collated_buffer = TRY(Core::AnonymousBuffer::create_with_size(total_buffer_size));
+
+    auto* write_pointer = collated_buffer.data<u8>();
+    for (auto const& bitmap_option : bitmaps) {
+        if (bitmap_option.has_value()) {
+            auto const& bitmap = bitmap_option.value();
+            memcpy(write_pointer, bitmap->scanline(0), bitmap->size_in_bytes());
+            write_pointer += bitmap->size_in_bytes();
+        }
+    }
+
+    TRY(encoder.encode(collated_buffer));
+
+    return {};
+}
+
+template<>
+ErrorOr<Gfx::BitmapSequence> decode(Decoder& decoder)
+{
+    auto metadata = TRY(decoder.decode<Vector<Optional<Gfx::BitmapMetadata>>>());
+    auto collated_buffer = TRY(decoder.decode<Core::AnonymousBuffer>());
+
+    Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+    bitmaps.ensure_capacity(metadata.size());
+
+    ReadonlyBytes bytes = ReadonlyBytes(collated_buffer.data<u8>(), collated_buffer.size());
+    size_t bytes_read = 0;
+
+    // sequentially read each valid bitmap's data from the collated buffer
+    for (auto const& metadata_option : metadata) {
+        Optional<NonnullRefPtr<Gfx::Bitmap>> bitmap = {};
+
+        if (metadata_option.has_value()) {
+            auto metadata = metadata_option.value();
+            size_t size_in_bytes = metadata.size_in_bytes;
+
+            if (bytes_read + size_in_bytes > bytes.size())
+                return Error::from_string_literal("IPC: Invalid Gfx::BitmapSequence buffer data");
+
+            auto buffer = TRY(Core::AnonymousBuffer::create_with_size(size_in_bytes));
+
+            memcpy(buffer.data<u8>(), bytes.slice(bytes_read, size_in_bytes).data(), size_in_bytes);
+            bytes_read += size_in_bytes;
+
+            bitmap = TRY(Gfx::Bitmap::create_with_anonymous_buffer(metadata.format, move(buffer), metadata.size, metadata.scale));
+        }
+
+        bitmaps.append(bitmap);
+    }
+
+    return Gfx::BitmapSequence { bitmaps };
+}
+
+}

Userland/Libraries/LibGfx/BitmapSequence.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2024, Zachary Huang <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/RefPtr.h>
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/Size.h>
+#include <LibIPC/Forward.h>
+
+namespace Gfx {
+
+struct BitmapSequence {
+    Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+};
+
+// a struct to temporarily store bitmap fields before the buffer data is decoded
+struct BitmapMetadata {
+    Gfx::BitmapFormat format;
+    Gfx::IntSize size;
+    int scale;
+    size_t size_in_bytes;
+};
+
+}
+
+namespace IPC {
+
+template<>
+ErrorOr<void> encode(Encoder&, Gfx::BitmapMetadata const&);
+
+template<>
+ErrorOr<Gfx::BitmapMetadata> decode(Decoder&);
+
+template<>
+ErrorOr<void> encode(Encoder&, Gfx::BitmapSequence const&);
+
+template<>
+ErrorOr<Gfx::BitmapSequence> decode(Decoder&);
+
+}

Userland/Libraries/LibGfx/CMakeLists.txt

@@ -3,6 +3,7 @@ set(SOURCES
     AntiAliasingPainter.cpp
     Bitmap.cpp
     BitmapMixer.cpp
+    BitmapSequence.cpp
     CMYKBitmap.cpp
     ClassicStylePainter.cpp
     ClassicWindowTheme.cpp

Userland/Libraries/LibImageDecoderClient/Client.cpp

@@ -60,8 +60,9 @@ NonnullRefPtr<Core::Promise<DecodedImage>> Client::decode_image(ReadonlyBytes en
     return promise;
 }
 
-void Client::did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> const& bitmaps, Vector<u32> const& durations, Gfx::FloatPoint scale)
+void Client::did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Gfx::BitmapSequence const& bitmap_sequence, Vector<u32> const& durations, Gfx::FloatPoint scale)
 {
+    auto const& bitmaps = bitmap_sequence.bitmaps;
     VERIFY(!bitmaps.is_empty());
 
     auto maybe_promise = m_pending_decoded_images.take(image_id);

Userland/Libraries/LibImageDecoderClient/Client.h

@@ -41,7 +41,7 @@ public:
 private:
     virtual void die() override;
 
-    virtual void did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> const& bitmaps, Vector<u32> const& durations, Gfx::FloatPoint scale) override;
+    virtual void did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Gfx::BitmapSequence const& bitmap_sequence, Vector<u32> const& durations, Gfx::FloatPoint scale) override;
     virtual void did_fail_to_decode_image(i64 image_id, String const& error_message) override;
 
     HashMap<i64, NonnullRefPtr<Core::Promise<DecodedImage>>> m_pending_decoded_images;

Userland/Services/ImageDecoder/ConnectionFromClient.cpp

@@ -58,6 +58,8 @@ static ErrorOr<ConnectionFromClient::DecodeResult> decode_image_to_details(Core:
     result.is_animated = decoder->is_animated();
     result.loop_count = decoder->loop_count();
 
+    Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+
     if (auto maybe_metadata = decoder->metadata(); maybe_metadata.has_value() && is<Gfx::ExifMetadata>(*maybe_metadata)) {
         auto const& exif = static_cast<Gfx::ExifMetadata const&>(maybe_metadata.value());
         if (exif.x_resolution().has_value() && exif.y_resolution().has_value()) {
@@ -70,11 +72,13 @@ static ErrorOr<ConnectionFromClient::DecodeResult> decode_image_to_details(Core:
         }
     }
 
-    decode_image_to_bitmaps_and_durations_with_decoder(*decoder, move(ideal_size), result.bitmaps, result.durations);
+    decode_image_to_bitmaps_and_durations_with_decoder(*decoder, move(ideal_size), bitmaps, result.durations);
 
-    if (result.bitmaps.is_empty())
+    if (bitmaps.is_empty())
         return Error::from_string_literal("Could not decode image");
 
+    result.bitmaps = Gfx::BitmapSequence { bitmaps };
+
     return result;
 }
 

Userland/Services/ImageDecoder/ConnectionFromClient.h

@@ -10,6 +10,7 @@
 #include <ImageDecoder/Forward.h>
 #include <ImageDecoder/ImageDecoderClientEndpoint.h>
 #include <ImageDecoder/ImageDecoderServerEndpoint.h>
+#include <LibGfx/BitmapSequence.h>
 #include <LibIPC/ConnectionFromClient.h>
 #include <LibThreading/BackgroundAction.h>
 
@@ -28,7 +29,7 @@ public:
         bool is_animated = false;
         u32 loop_count = 0;
         Gfx::FloatPoint scale { 1, 1 };
-        Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps;
+        Gfx::BitmapSequence bitmaps;
         Vector<u32> durations;
     };
 

Userland/Services/ImageDecoder/ImageDecoderClient.ipc

@@ -1,7 +1,7 @@
-#include <LibGfx/ShareableBitmap.h>
+#include <LibGfx/BitmapSequence.h>
 
 endpoint ImageDecoderClient
 {
-    did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Vector<Optional<NonnullRefPtr<Gfx::Bitmap>>> bitmaps, Vector<u32> durations, Gfx::FloatPoint scale) =|
+    did_decode_image(i64 image_id, bool is_animated, u32 loop_count, Gfx::BitmapSequence bitmaps, Vector<u32> durations, Gfx::FloatPoint scale) =|
     did_fail_to_decode_image(i64 image_id, String error_message) =|
 }