PixelPaint: Ensure the selection is always within image bounds

8 files changed, 97 insertions, 55 deletions

diff --git a/Userland/Applications/PixelPaint/MainWidget.cpp b/Userland/Applications/PixelPaint/MainWidget.cpp
index c6f45430c6..02c6efe26b 100644
--- a/Userland/Applications/PixelPaint/MainWidget.cpp
+++ b/Userland/Applications/PixelPaint/MainWidget.cpp
@@ -418,7 +418,8 @@ ErrorOr<void> MainWidget::initialize_menubar(GUI::Window& window)
         VERIFY(editor);
         if (!editor->active_layer())
             return;
-        editor->image().selection().merge(editor->active_layer()->relative_rect(), PixelPaint::Selection::MergeMode::Set);
+        auto layer_rect = editor->active_layer()->relative_rect();
+        editor->image().selection().merge(layer_rect.intersected(editor->image().rect()), PixelPaint::Selection::MergeMode::Set);
         editor->did_complete_action("Select All"sv);
     })));
     TRY(m_edit_menu->try_add_action(GUI::Action::create(
@@ -660,6 +661,7 @@ ErrorOr<void> MainWidget::initialize_menubar(GUI::Window& window)
                     GUI::MessageBox::show_error(&window, MUST(String::formatted("Failed to resize image: {}", image_resize_or_error.release_error())));
                     return;
                 }
+                // FIXME: We should ensure the selection is within the bounds of the image here.
                 editor->did_complete_action("Resize Image"sv);
             }
         })));
diff --git a/Userland/Applications/PixelPaint/Selection.cpp b/Userland/Applications/PixelPaint/Selection.cpp
index b936ac9420..760549cc29 100644
--- a/Userland/Applications/PixelPaint/Selection.cpp
+++ b/Userland/Applications/PixelPaint/Selection.cpp
@@ -31,6 +31,8 @@ void Selection::invert()
 
 void Selection::merge(Mask const& mask, MergeMode mode)
 {
+    VERIFY(m_image.rect().contains(mask.bounding_rect()));
+
     switch (mode) {
     case MergeMode::Set:
         m_mask = mask;
diff --git a/Userland/Applications/PixelPaint/Tools/LassoSelectTool.cpp b/Userland/Applications/PixelPaint/Tools/LassoSelectTool.cpp
index 54751ef8b5..24aeb6beb2 100644
--- a/Userland/Applications/PixelPaint/Tools/LassoSelectTool.cpp
+++ b/Userland/Applications/PixelPaint/Tools/LassoSelectTool.cpp
@@ -72,11 +72,20 @@ void LassoSelectTool::on_mouseup(Layer*, MouseEvent&)
     m_selecting = false;
     m_top_left.translate_by(-1);
 
+    auto image_rect = m_editor->image().rect();
+    auto lasso_rect = Gfx::IntRect::from_two_points(m_top_left, m_bottom_right);
+    if (!lasso_rect.intersects(image_rect)) {
+        m_editor->image().selection().merge(Gfx::IntRect {}, m_merge_mode);
+        return;
+    }
+
     if (m_path_points.last() != m_start_position)
         m_path_points.append(m_start_position);
 
     // We create a bitmap that is bigger by 1 pixel on each side
-    auto lasso_bitmap_rect = Gfx::IntRect::from_two_points(m_top_left, m_bottom_right).inflated(2, 2);
+    auto lasso_bitmap_rect = lasso_rect.inflated(2, 2);
+    // FIXME: It should be possible to limit the size of the lasso bitmap to the size of the canvas, as that is
+    //        the maximum possible size of the selection.
     auto lasso_bitmap_or_error = Gfx::Bitmap::create(Gfx::BitmapFormat::BGRA8888, lasso_bitmap_rect.size());
     if (lasso_bitmap_or_error.is_error())
         return;
@@ -97,9 +106,13 @@ void LassoSelectTool::flood_lasso_selection(Gfx::Bitmap& lasso_bitmap)
     VERIFY(lasso_bitmap.bpp() == 32);
 
     // Create Mask which will track already-processed pixels
-    auto selection_mask = Mask::full({ m_top_left, lasso_bitmap.size() });
+    auto mask_rect = Gfx::IntRect(m_top_left, lasso_bitmap.size()).intersected(m_editor->image().rect());
+    auto selection_mask = Mask::full(mask_rect);
+
     auto pixel_reached = [&](Gfx::IntPoint location) {
-        selection_mask.set(Gfx::IntPoint(m_top_left.x() + location.x(), m_top_left.y() + location.y()), 0);
+        auto point_to_set = location.translated(m_top_left);
+        if (mask_rect.contains(point_to_set))
+            selection_mask.set(point_to_set, 0);
     };
 
     lasso_bitmap.flood_visit_from_point({ 0, 0 }, 0, move(pixel_reached));
diff --git a/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.cpp b/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.cpp
index 94063e422d..04daeafd4b 100644
--- a/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.cpp
+++ b/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.cpp
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2022-2023, the SerenityOS developers.
+ * Copyright (c) 2023, Tim Ledbetter <timledbetter@gmail.com>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -24,14 +25,16 @@ void PolygonalSelectTool::flood_polygon_selection(Gfx::Bitmap& polygon_bitmap, G
     VERIFY(polygon_bitmap.bpp() == 32);
 
     // Create Mask which will track already-processed pixels.
-    Mask selection_mask = Mask::full(polygon_bitmap.rect().translated(polygon_delta));
+    auto mask_rect = Gfx::IntRect(polygon_delta, polygon_bitmap.size()).intersected(m_editor->image().rect());
+    auto selection_mask = Mask::full(mask_rect);
 
     auto pixel_reached = [&](Gfx::IntPoint location) {
-        selection_mask.set(Gfx::IntPoint(location.x(), location.y()).translated(polygon_delta), 0);
+        auto point_to_set = location.translated(polygon_delta);
+        if (mask_rect.contains(point_to_set))
+            selection_mask.set(point_to_set, 0);
     };
 
-    polygon_bitmap.flood_visit_from_point({ polygon_bitmap.width() - 1, polygon_bitmap.height() - 1 }, 0, move(pixel_reached));
-
+    polygon_bitmap.flood_visit_from_point({ 0, 0 }, 0, move(pixel_reached));
     selection_mask.shrink_to_fit();
     m_editor->image().selection().merge(selection_mask, m_merge_mode);
 }
@@ -39,60 +42,65 @@ void PolygonalSelectTool::flood_polygon_selection(Gfx::Bitmap& polygon_bitmap, G
 void PolygonalSelectTool::process_polygon()
 {
     // Determine minimum bounding box that can hold the polygon.
-    auto min_x_seen = m_polygon_points.at(0).x();
-    auto max_x_seen = m_polygon_points.at(0).x();
-    auto min_y_seen = m_polygon_points.at(0).y();
-    auto max_y_seen = m_polygon_points.at(0).y();
+    auto top_left = m_polygon_points.at(0);
+    auto bottom_right = m_polygon_points.at(0);
 
     for (auto point : m_polygon_points) {
-        if (point.x() < min_x_seen)
-            min_x_seen = point.x();
-        if (point.x() > max_x_seen)
-            max_x_seen = point.x();
-        if (point.y() < min_y_seen)
-            min_y_seen = point.y();
-        if (point.y() > max_y_seen)
-            max_y_seen = point.y();
+        if (point.x() < top_left.x())
+            top_left.set_x(point.x());
+        if (point.x() > bottom_right.x())
+            bottom_right.set_x(point.x());
+        if (point.y() < top_left.y())
+            top_left.set_y(point.y());
+        if (point.y() > bottom_right.y())
+            bottom_right.set_y(point.y());
     }
 
-    // We create a bitmap that is bigger by 1 pixel on each side (+2) and need to account for the 0 indexed
-    // pixel positions (+1) so we make the bitmap size the delta of x/y min/max + 3.
-    auto polygon_bitmap_or_error = Gfx::Bitmap::create(Gfx::BitmapFormat::BGRA8888, { (max_x_seen - min_x_seen) + 3, (max_y_seen - min_y_seen) + 3 });
-    if (polygon_bitmap_or_error.is_error())
+    top_left.translate_by(-1);
+    auto polygon_rect = Gfx::IntRect::from_two_points(top_left, bottom_right);
+    auto image_rect = m_editor->image().rect();
+    if (!polygon_rect.intersects(image_rect)) {
+        m_editor->image().selection().merge(Gfx::IntRect {}, m_merge_mode);
         return;
+    }
 
-    auto polygon_bitmap = polygon_bitmap_or_error.release_value();
+    if (m_polygon_points.last() != m_polygon_points.first())
+        m_polygon_points.append(m_polygon_points.first());
 
-    auto polygon_painter = Gfx::Painter(polygon_bitmap);
     // We want to paint the polygon into the bitmap such that there is an empty 1px border all the way around it
-    // this ensures that we have a known pixel (0,0) that is outside the polygon. Since the coordinates are relative
-    // to the layer but the bitmap is cropped to the bounding rect of the polygon we need to offset our
-    // points by the the negative of min x/y. And because we want a 1 px offset to the right and down, we + 1 this.
-    auto polygon_bitmap_delta = Gfx::IntPoint(-min_x_seen + 1, -min_y_seen + 1);
-    polygon_painter.translate(polygon_bitmap_delta);
+    // this ensures that we have a known pixel (0,0) that is outside the polygon.
+    auto bitmap_rect = polygon_rect.inflated(2, 2);
+    // FIXME: It should be possible to limit the size of the polygon bitmap to the size of the canvas, as that is
+    //        the maximum possible size of the selection.
+    auto polygon_bitmap_or_error = Gfx::Bitmap::create(Gfx::BitmapFormat::BGRA8888, bitmap_rect.size());
+    if (polygon_bitmap_or_error.is_error())
+        return;
+
+    auto polygon_bitmap = polygon_bitmap_or_error.release_value();
+    Gfx::Painter polygon_painter(polygon_bitmap);
     for (size_t i = 0; i < m_polygon_points.size() - 1; i++) {
-        polygon_painter.draw_line(m_polygon_points.at(i), m_polygon_points.at(i + 1), Color::Black);
+        auto line_start = m_polygon_points.at(i) - top_left;
+        auto line_end = m_polygon_points.at(i + 1) - top_left;
+        polygon_painter.draw_line(line_start, line_end, Color::Black);
     }
-    polygon_painter.draw_line(m_polygon_points.at(m_polygon_points.size() - 1), m_polygon_points.at(0), Color::Black);
 
-    // Delta to use for mapping the bitmap back to layer coordinates. -1 to account for the right and down offset.
-    auto bitmap_to_layer_delta = Gfx::IntPoint(min_x_seen + m_editor->active_layer()->location().x() - 1, min_y_seen + m_editor->active_layer()->location().y() - 1);
-    flood_polygon_selection(polygon_bitmap, bitmap_to_layer_delta);
+    flood_polygon_selection(polygon_bitmap, top_left);
 }
+
 void PolygonalSelectTool::on_mousedown(Layer*, MouseEvent& event)
 {
-    auto& image_event = event.image_event();
+    auto const& image_event = event.image_event();
     if (image_event.button() != GUI::MouseButton::Primary)
         return;
     if (!m_selecting) {
         m_polygon_points.clear();
-        m_last_selecting_cursor_position = event.layer_event().position();
+        m_last_selecting_cursor_position = image_event.position();
     }
 
     m_selecting = true;
 
-    auto new_point = event.layer_event().position();
-    if (!m_polygon_points.is_empty() && event.layer_event().shift())
+    auto new_point = image_event.position();
+    if (!m_polygon_points.is_empty() && image_event.shift())
         new_point = Tool::constrain_line_angle(m_polygon_points.last(), new_point);
 
     // This point matches the first point exactly. Consider this polygon finished.
@@ -120,10 +128,11 @@ void PolygonalSelectTool::on_mousemove(Layer*, MouseEvent& event)
     if (!m_selecting)
         return;
 
-    if (event.layer_event().shift())
-        m_last_selecting_cursor_position = Tool::constrain_line_angle(m_polygon_points.last(), event.layer_event().position());
+    auto const& image_event = event.image_event();
+    if (image_event.shift())
+        m_last_selecting_cursor_position = Tool::constrain_line_angle(m_polygon_points.last(), image_event.position());
     else
-        m_last_selecting_cursor_position = event.layer_event().position();
+        m_last_selecting_cursor_position = image_event.position();
 
     m_editor->update();
 }
@@ -137,7 +146,7 @@ void PolygonalSelectTool::on_doubleclick(Layer*, MouseEvent&)
     m_editor->update();
 }
 
-void PolygonalSelectTool::on_second_paint(Layer const* layer, GUI::PaintEvent& event)
+void PolygonalSelectTool::on_second_paint(Layer const*, GUI::PaintEvent& event)
 {
     if (!m_selecting)
         return;
@@ -145,8 +154,6 @@ void PolygonalSelectTool::on_second_paint(Layer const* layer, GUI::PaintEvent& e
     GUI::Painter painter(*m_editor);
     painter.add_clip_rect(event.rect());
 
-    painter.translate(editor_layer_location(*layer));
-
     auto draw_preview_lines = [&](auto color, auto thickness) {
         for (size_t i = 0; i < m_polygon_points.size() - 1; i++) {
             auto preview_start = editor_stroke_position(m_polygon_points.at(i), 1);
diff --git a/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.h b/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.h
index 68677a5e8a..32e6666e99 100644
--- a/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.h
+++ b/Userland/Applications/PixelPaint/Tools/PolygonalSelectTool.h
@@ -27,7 +27,7 @@ public:
     virtual Gfx::IntPoint point_position_to_preferred_cell(Gfx::FloatPoint position) const override;
 
 private:
-    virtual void flood_polygon_selection(Gfx::Bitmap&, Gfx::IntPoint);
+    virtual void flood_polygon_selection(Gfx::Bitmap&, Gfx::IntPoint polygon_delta);
     virtual void process_polygon();
     virtual StringView tool_name() const override { return "Polygonal Select Tool"sv; }
 
diff --git a/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.cpp b/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.cpp
index e8415b4cdf..7428b20cf7 100644
--- a/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.cpp
+++ b/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.cpp
@@ -62,7 +62,8 @@ void RectangleSelectTool::on_mouseup(Layer*, MouseEvent& event)
 
     m_editor->update();
 
-    auto rect_in_image = Gfx::IntRect::from_two_points(m_selection_start, m_selection_end);
+    auto rect_in_image = selection_rect();
+
     auto mask = Mask::full(rect_in_image);
 
     auto feathering = ((mask.bounding_rect().size().to_type<float>() * .5f) * m_edge_feathering).to_type<int>();
@@ -141,7 +142,10 @@ void RectangleSelectTool::on_second_paint(Layer const*, GUI::PaintEvent& event)
     GUI::Painter painter(*m_editor);
     painter.add_clip_rect(event.rect());
 
-    auto rect_in_image = Gfx::IntRect::from_two_points(m_selection_start, m_selection_end);
+    auto rect_in_image = selection_rect();
+    if (rect_in_image.is_empty())
+        return;
+
     auto rect_in_editor = m_editor->content_to_frame_rect(rect_in_image);
 
     m_editor->draw_marching_ants(painter, rect_in_editor.to_rounded<int>());
@@ -223,4 +227,14 @@ Gfx::IntPoint RectangleSelectTool::point_position_to_preferred_cell(Gfx::FloatPo
     return position.to_rounded<int>();
 }
 
+Gfx::IntRect RectangleSelectTool::selection_rect() const
+{
+    auto image_rect = m_editor->image().rect();
+    auto unconstrained_selection_rect = Gfx::IntRect::from_two_points(m_selection_start, m_selection_end);
+    if (!unconstrained_selection_rect.intersects(image_rect))
+        return {};
+
+    return unconstrained_selection_rect.intersected(image_rect);
+}
+
 }
diff --git a/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.h b/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.h
index 7bbb7eae96..168d8e9fca 100644
--- a/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.h
+++ b/Userland/Applications/PixelPaint/Tools/RectangleSelectTool.h
@@ -47,6 +47,8 @@ private:
     MovingMode m_moving_mode { MovingMode::None };
     Gfx::IntPoint m_selection_start;
     Gfx::IntPoint m_selection_end;
+
+    Gfx::IntRect selection_rect() const;
 };
 
 }
diff --git a/Userland/Applications/PixelPaint/Tools/WandSelectTool.cpp b/Userland/Applications/PixelPaint/Tools/WandSelectTool.cpp
index c00144c294..0806d08075 100644
--- a/Userland/Applications/PixelPaint/Tools/WandSelectTool.cpp
+++ b/Userland/Applications/PixelPaint/Tools/WandSelectTool.cpp
@@ -21,14 +21,18 @@
 
 namespace PixelPaint {
 
-static void set_flood_selection(Gfx::Bitmap& bitmap, Image& image, Gfx::IntPoint start_position, Gfx::IntPoint selection_offset, int threshold, Selection::MergeMode merge_mode)
+static void set_flood_selection(Gfx::Bitmap& bitmap, Image& image, Gfx::IntPoint start_position, Gfx::IntRect layer_rect, int threshold, Selection::MergeMode merge_mode)
 {
     VERIFY(bitmap.bpp() == 32);
 
-    auto selection_mask = Mask::empty({ selection_offset, bitmap.size() });
+    auto image_rect = image.rect();
+    auto mask_rect = layer_rect.intersected(image_rect);
+    auto selection_mask = Mask::empty(mask_rect);
 
     auto pixel_reached = [&](Gfx::IntPoint location) {
-        selection_mask.set(selection_offset.x() + location.x(), selection_offset.y() + location.y(), 0xFF);
+        auto point_to_set = layer_rect.top_left() + location;
+        if (selection_mask.bounding_rect().contains(point_to_set))
+            selection_mask.set(point_to_set, 0xFF);
     };
 
     bitmap.flood_visit_from_point(start_position, threshold, move(pixel_reached));
@@ -55,10 +59,8 @@ void WandSelectTool::on_mousedown(Layer* layer, MouseEvent& event)
     if (!layer->rect().contains(layer_event.position()))
         return;
 
-    auto selection_offset = layer->relative_rect().top_left();
-
     m_editor->image().selection().begin_interactive_selection();
-    set_flood_selection(layer->currently_edited_bitmap(), m_editor->image(), layer_event.position(), selection_offset, m_threshold, m_merge_mode);
+    set_flood_selection(layer->currently_edited_bitmap(), m_editor->image(), layer_event.position(), layer->relative_rect(), m_threshold, m_merge_mode);
     m_editor->image().selection().end_interactive_selection();
     m_editor->update();
     m_editor->did_complete_action(tool_name());