PixelPaint: Make image flattening actions work with disjoint layers

The "Flatten Image" and "Merge Visible" actions now work with layers
of different sizes. These actions now expand the bounding rect of the
newly merged layer to contain all layers being merged.

3 files changed, 61 insertions, 38 deletions

diff --git a/Userland/Applications/PixelPaint/Image.cpp b/Userland/Applications/PixelPaint/Image.cpp
index 7d0f5dfb18..dbc916aef1 100644
--- a/Userland/Applications/PixelPaint/Image.cpp
+++ b/Userland/Applications/PixelPaint/Image.cpp
@@ -334,50 +334,60 @@ void Image::remove_layer(Layer& layer)
     did_modify_layer_stack();
 }
 
-void Image::flatten_all_layers()
+ErrorOr<void> Image::flatten_all_layers()
 {
-    if (m_layers.size() < 2)
-        return;
-
-    auto& bottom_layer = m_layers.at(0);
-
-    GUI::Painter painter(bottom_layer.content_bitmap());
-    paint_into(painter, { 0, 0, m_size.width(), m_size.height() }, 1.0f);
+    return merge_layers(LayerMergeMode::All);
+}
 
-    for (size_t index = m_layers.size() - 1; index > 0; index--) {
-        auto& layer = m_layers.at(index);
-        remove_layer(layer);
-    }
-    bottom_layer.set_name("Background");
-    select_layer(&bottom_layer);
+ErrorOr<void> Image::merge_visible_layers()
+{
+    return merge_layers(LayerMergeMode::VisibleOnly);
 }
 
-void Image::merge_visible_layers()
+ErrorOr<void> Image::merge_layers(LayerMergeMode layer_merge_mode)
 {
     if (m_layers.size() < 2)
-        return;
-
-    size_t index = 0;
+        return {};
 
-    while (index < m_layers.size()) {
-        if (m_layers.at(index).is_visible()) {
-            auto& bottom_layer = m_layers.at(index);
-            GUI::Painter painter(bottom_layer.content_bitmap());
-            paint_into(painter, { 0, 0, m_size.width(), m_size.height() }, 1.0f);
-            select_layer(&bottom_layer);
-            index++;
-            break;
+    NonnullRefPtrVector<Layer> new_layers;
+    Gfx::IntRect merged_layer_bounding_rect = {};
+    size_t bottom_layer_index = 0;
+    for (auto const& layer : m_layers) {
+        if (!layer.is_visible()) {
+            if (layer_merge_mode == LayerMergeMode::VisibleOnly)
+                TRY(new_layers.try_append(layer));
+            if (merged_layer_bounding_rect.is_empty())
+                bottom_layer_index++;
+            continue;
         }
-        index++;
+        merged_layer_bounding_rect = merged_layer_bounding_rect.united(layer.relative_rect());
     }
-    while (index < m_layers.size()) {
-        if (m_layers.at(index).is_visible()) {
-            auto& layer = m_layers.at(index);
-            remove_layer(layer);
-        } else {
-            index++;
-        }
+
+    if (merged_layer_bounding_rect.is_empty())
+        return {};
+
+    NonnullRefPtr<Layer> bottom_layer = m_layers.at(bottom_layer_index);
+    NonnullRefPtr<Layer> merged_layer = bottom_layer;
+    if (!merged_layer->relative_rect().contains(merged_layer_bounding_rect)) {
+        merged_layer = TRY(Layer::create_with_size(*this, merged_layer_bounding_rect.size(), bottom_layer->name()));
+        merged_layer->set_location(merged_layer_bounding_rect.location());
     }
+
+    GUI::Painter painter(merged_layer->content_bitmap());
+    if (merged_layer.ptr() != bottom_layer.ptr())
+        painter.blit(bottom_layer->location() - merged_layer->location(), bottom_layer->display_bitmap(), bottom_layer->rect(), static_cast<float>(bottom_layer->opacity_percent()) / 100.0f);
+    for (size_t index = bottom_layer_index + 1; index < m_layers.size(); index++) {
+        auto& layer = m_layers.at(index);
+        if (!layer.is_visible())
+            continue;
+        painter.blit(layer.location() - merged_layer->location(), layer.display_bitmap(), layer.rect(), static_cast<float>(layer.opacity_percent()) / 100.0f);
+    }
+
+    TRY(new_layers.try_append(merged_layer));
+    m_layers = move(new_layers);
+    select_layer(merged_layer.ptr());
+    did_modify_layer_stack();
+    return {};
 }
 
 void Image::merge_active_layer_up(Layer& layer)
diff --git a/Userland/Applications/PixelPaint/Image.h b/Userland/Applications/PixelPaint/Image.h
index 41cba3f113..4a4a154d3d 100644
--- a/Userland/Applications/PixelPaint/Image.h
+++ b/Userland/Applications/PixelPaint/Image.h
@@ -83,8 +83,8 @@ public:
     void change_layer_index(size_t old_index, size_t new_index);
     void remove_layer(Layer&);
     void select_layer(Layer*);
-    void flatten_all_layers();
-    void merge_visible_layers();
+    ErrorOr<void> flatten_all_layers();
+    ErrorOr<void> merge_visible_layers();
     void merge_active_layer_up(Layer& layer);
     void merge_active_layer_down(Layer& layer);
 
@@ -106,12 +106,19 @@ public:
     Color color_at(Gfx::IntPoint point) const;
 
 private:
+    enum class LayerMergeMode {
+        All,
+        VisibleOnly
+    };
+
     explicit Image(Gfx::IntSize);
 
     void did_change(Gfx::IntRect const& modified_rect = {});
     void did_change_rect(Gfx::IntRect const& modified_rect = {});
     void did_modify_layer_stack();
 
+    ErrorOr<void> merge_layers(LayerMergeMode);
+
     Gfx::IntSize m_size;
     NonnullRefPtrVector<Layer> m_layers;
 
diff --git a/Userland/Applications/PixelPaint/MainWidget.cpp b/Userland/Applications/PixelPaint/MainWidget.cpp
index 8978249a92..30f4d8cb48 100644
--- a/Userland/Applications/PixelPaint/MainWidget.cpp
+++ b/Userland/Applications/PixelPaint/MainWidget.cpp
@@ -845,7 +845,10 @@ ErrorOr<void> MainWidget::initialize_menubar(GUI::Window& window)
         "Fl&atten Image", { Mod_Ctrl, Key_F }, g_icon_bag.flatten_image, [&](auto&) {
             auto* editor = current_image_editor();
             VERIFY(editor);
-            editor->image().flatten_all_layers();
+            if (auto maybe_error = editor->image().flatten_all_layers(); maybe_error.is_error()) {
+                GUI::MessageBox::show_error(&window, DeprecatedString::formatted("Failed to flatten all layers: {}", maybe_error.release_error()));
+                return;
+            }
             editor->did_complete_action("Flatten Image"sv);
         }));
 
@@ -853,7 +856,10 @@ ErrorOr<void> MainWidget::initialize_menubar(GUI::Window& window)
         "&Merge Visible", { Mod_Ctrl, Key_M }, g_icon_bag.merge_visible, [&](auto&) {
             auto* editor = current_image_editor();
             VERIFY(editor);
-            editor->image().merge_visible_layers();
+            if (auto maybe_error = editor->image().merge_visible_layers(); maybe_error.is_error()) {
+                GUI::MessageBox::show_error(&window, DeprecatedString::formatted("Failed to merge visible layers: {}", maybe_error.release_error()));
+                return;
+            }
             editor->did_complete_action("Merge Visible"sv);
         }));