LibGfx: Don't allow creating bitmaps whose sizes would overflow

If the area or size_in_bytes calculation for a Gfx::Bitmap would
overflow, we now refuse to create such a bitmap and return nullptr.

Thanks to @itamar8910 for finding this! :^)

2 files changed, 8 insertions, 4 deletions

diff --git a/Libraries/LibGUI/Window.cpp b/Libraries/LibGUI/Window.cpp
index 5fe3b255e3..87a1e25ab8 100644
--- a/Libraries/LibGUI/Window.cpp
+++ b/Libraries/LibGUI/Window.cpp
@@ -251,10 +251,12 @@ void Window::event(Core::Event& event)
         bool created_new_backing_store = !m_back_bitmap;
         if (!m_back_bitmap) {
             m_back_bitmap = create_backing_bitmap(paint_event.window_size());
+            ASSERT(m_back_bitmap);
         } else if (m_double_buffering_enabled) {
             bool still_has_pixels = m_back_bitmap->shared_buffer()->set_nonvolatile();
             if (!still_has_pixels) {
                 m_back_bitmap = create_backing_bitmap(paint_event.window_size());
+                ASSERT(m_back_bitmap);
                 created_new_backing_store = true;
             }
         }
@@ -514,6 +516,7 @@ void Window::flip(const Vector<Gfx::Rect, 32>& dirty_rects)
 
     if (!m_back_bitmap || m_back_bitmap->size() != m_front_bitmap->size()) {
         m_back_bitmap = create_backing_bitmap(m_front_bitmap->size());
+        ASSERT(m_back_bitmap);
         memcpy(m_back_bitmap->scanline(0), m_front_bitmap->scanline(0), m_front_bitmap->size_in_bytes());
         m_back_bitmap->shared_buffer()->set_volatile();
         return;
@@ -527,7 +530,7 @@ void Window::flip(const Vector<Gfx::Rect, 32>& dirty_rects)
     m_back_bitmap->shared_buffer()->set_volatile();
 }
 
-NonnullRefPtr<Gfx::Bitmap> Window::create_shared_bitmap(Gfx::BitmapFormat format, const Gfx::Size& size)
+RefPtr<Gfx::Bitmap> Window::create_shared_bitmap(Gfx::BitmapFormat format, const Gfx::Size& size)
 {
     ASSERT(WindowServerConnection::the().server_pid());
     ASSERT(!size.is_empty());
@@ -539,7 +542,7 @@ NonnullRefPtr<Gfx::Bitmap> Window::create_shared_bitmap(Gfx::BitmapFormat format
     return Gfx::Bitmap::create_with_shared_buffer(format, *shared_buffer, size);
 }
 
-NonnullRefPtr<Gfx::Bitmap> Window::create_backing_bitmap(const Gfx::Size& size)
+RefPtr<Gfx::Bitmap> Window::create_backing_bitmap(const Gfx::Size& size)
 {
     auto format = m_has_alpha_channel ? Gfx::BitmapFormat::RGBA32 : Gfx::BitmapFormat::RGB32;
     return create_shared_bitmap(format, size);
@@ -561,6 +564,7 @@ void Window::set_icon(const Gfx::Bitmap* icon)
         return;
 
     m_icon = create_shared_bitmap(Gfx::BitmapFormat::RGBA32, icon->size());
+    ASSERT(m_icon);
     {
         Painter painter(*m_icon);
         painter.blit({ 0, 0 }, *icon, icon->rect());
diff --git a/Libraries/LibGUI/Window.h b/Libraries/LibGUI/Window.h
index f8fe54652d..59d20a1bcd 100644
--- a/Libraries/LibGUI/Window.h
+++ b/Libraries/LibGUI/Window.h
@@ -189,8 +189,8 @@ protected:
 private:
     virtual bool is_window() const override final { return true; }
 
-    NonnullRefPtr<Gfx::Bitmap> create_backing_bitmap(const Gfx::Size&);
-    NonnullRefPtr<Gfx::Bitmap> create_shared_bitmap(Gfx::BitmapFormat, const Gfx::Size&);
+    RefPtr<Gfx::Bitmap> create_backing_bitmap(const Gfx::Size&);
+    RefPtr<Gfx::Bitmap> create_shared_bitmap(Gfx::BitmapFormat, const Gfx::Size&);
     void set_current_backing_bitmap(Gfx::Bitmap&, bool flush_immediately = false);
     void flip(const Vector<Gfx::Rect, 32>& dirty_rects);
     void force_update();