WindowServer: Improve screen invalidation on window state changes

Because window states and various flags can affect the windows'
rendered areas it's safer to use the last computed occlusion rectangles
to invalidate areas on the screen that may have to be re-rendered due
to e.g. a window size change.

Fixes #6723

6 files changed, 85 insertions, 45 deletions

diff --git a/Userland/Services/WindowServer/Compositor.cpp b/Userland/Services/WindowServer/Compositor.cpp
index dfc957dee1..32cd034087 100644
--- a/Userland/Services/WindowServer/Compositor.cpp
+++ b/Userland/Services/WindowServer/Compositor.cpp
@@ -697,6 +697,18 @@ void Compositor::invalidate_screen(const Gfx::IntRect& screen_rect)
     start_compose_async_timer();
 }
 
+void Compositor::invalidate_screen(Gfx::DisjointRectSet const& rects)
+{
+    m_dirty_screen_rects.add(rects.intersected(Screen::bounding_rect()));
+
+    if (m_invalidated_any)
+        return;
+
+    m_invalidated_any = true;
+    m_invalidated_window = true;
+    start_compose_async_timer();
+}
+
 void Compositor::invalidate_window()
 {
     if (m_invalidated_window)
@@ -1089,14 +1101,31 @@ void Compositor::recompute_occlusions()
         visible_rects.add_many(Screen::rects());
         bool have_transparent = false;
         wm.for_each_visible_window_from_front_to_back([&](Window& w) {
+            VERIFY(!w.is_minimized());
             w.transparency_wallpaper_rects().clear();
+            auto previous_visible_opaque = move(w.opaque_rects());
+            auto previous_visible_transparency = move(w.transparency_rects());
+
+            auto invalidate_previous_render_rects = [&](Gfx::IntRect const& new_render_rect) {
+                if (!previous_visible_opaque.is_empty()) {
+                    if (new_render_rect.is_empty())
+                        invalidate_screen(previous_visible_opaque);
+                    else
+                        invalidate_screen(previous_visible_opaque.shatter(new_render_rect));
+                }
+                if (!previous_visible_transparency.is_empty()) {
+                    if (new_render_rect.is_empty())
+                        invalidate_screen(previous_visible_transparency);
+                    else
+                        invalidate_screen(previous_visible_transparency.shatter(new_render_rect));
+                }
+            };
+
             auto& visible_opaque = w.opaque_rects();
-            visible_opaque.clear();
             auto& transparency_rects = w.transparency_rects();
-            transparency_rects.clear();
+            bool should_invalidate_old = w.should_invalidate_last_rendered_screen_rects();
+
             w.screens().clear_with_capacity();
-            if (w.is_minimized())
-                return IterationDecision::Continue;
 
             auto transition_offset = window_transition_offset(w);
             auto transparent_frame_render_rects = w.frame().transparent_render_rects();
@@ -1105,6 +1134,12 @@ void Compositor::recompute_occlusions()
                 transparent_frame_render_rects.translate_by(transition_offset);
                 opaque_frame_render_rects.translate_by(transition_offset);
             }
+            if (should_invalidate_old) {
+                for (auto& rect : opaque_frame_render_rects.rects())
+                    invalidate_previous_render_rects(rect);
+                for (auto& rect : transparent_frame_render_rects.rects())
+                    invalidate_previous_render_rects(rect);
+            }
             Gfx::DisjointRectSet visible_opaque_rects;
             Screen::for_each([&](auto& screen) {
                 auto screen_rect = screen.rect();
@@ -1138,8 +1173,7 @@ void Compositor::recompute_occlusions()
                     return IterationDecision::Continue;
                 }
 
-                if (w2.is_minimized())
-                    return IterationDecision::Continue;
+                VERIFY(!w2.is_minimized());
 
                 auto w2_render_rect = w2.frame().render_rect();
                 auto w2_render_rect_on_screen = w2_render_rect;
@@ -1245,10 +1279,8 @@ void Compositor::recompute_occlusions()
             // Determine what transparent window areas need to render the wallpaper first
             wm.for_each_visible_window_from_back_to_front([&](Window& w) {
                 auto& transparency_wallpaper_rects = w.transparency_wallpaper_rects();
-                if (w.is_minimized()) {
-                    transparency_wallpaper_rects.clear();
-                    return IterationDecision::Continue;
-                }
+                VERIFY(!w.is_minimized());
+
                 Gfx::DisjointRectSet& transparency_rects = w.transparency_rects();
                 if (transparency_rects.is_empty()) {
                     transparency_wallpaper_rects.clear();
diff --git a/Userland/Services/WindowServer/Compositor.h b/Userland/Services/WindowServer/Compositor.h
index d1a33da67f..2c8be4aa9a 100644
--- a/Userland/Services/WindowServer/Compositor.h
+++ b/Userland/Services/WindowServer/Compositor.h
@@ -42,6 +42,7 @@ public:
     void invalidate_window();
     void invalidate_screen();
     void invalidate_screen(const Gfx::IntRect&);
+    void invalidate_screen(Gfx::DisjointRectSet const&);
 
     void screen_resolution_changed();
 
diff --git a/Userland/Services/WindowServer/Window.cpp b/Userland/Services/WindowServer/Window.cpp
index b5f52d9006..8ebfbe44c2 100644
--- a/Userland/Services/WindowServer/Window.cpp
+++ b/Userland/Services/WindowServer/Window.cpp
@@ -141,7 +141,8 @@ void Window::set_rect(const Gfx::IntRect& rect)
     }
 
     invalidate(true, old_rect.size() != rect.size());
-    m_frame.window_rect_changed(old_rect, rect); // recomputes occlusions
+    m_frame.window_rect_changed(old_rect, rect);
+    invalidate_last_rendered_screen_rects();
 }
 
 void Window::set_rect_without_repaint(const Gfx::IntRect& rect)
@@ -161,7 +162,8 @@ void Window::set_rect_without_repaint(const Gfx::IntRect& rect)
     }
 
     invalidate(true, old_rect.size() != rect.size());
-    m_frame.window_rect_changed(old_rect, rect); // recomputes occlusions
+    m_frame.window_rect_changed(old_rect, rect);
+    invalidate_last_rendered_screen_rects();
 }
 
 bool Window::apply_minimum_size(Gfx::IntRect& rect)
@@ -284,12 +286,16 @@ void Window::set_minimized(bool minimized)
         return;
     m_minimized_state = minimized ? WindowMinimizedState::Minimized : WindowMinimizedState::None;
     update_window_menu_items();
-    Compositor::the().invalidate_occlusions();
-    Compositor::the().invalidate_screen(frame().render_rect());
+
     if (!blocking_modal_window())
         start_minimize_animation();
     if (!minimized)
         request_update({ {}, size() });
+
+    // Since a minimized window won't be visible we need to invalidate the last rendered
+    // rectangles before the next occlusion calculation
+    invalidate_last_rendered_screen_rects_now();
+
     WindowManager::the().notify_minimization_state_changed(*this);
 }
 
@@ -583,22 +589,24 @@ void Window::set_visible(bool b)
     if (!m_visible)
         WindowManager::the().check_hide_geometry_overlay(*this);
     Compositor::the().invalidate_occlusions();
-    if (m_visible)
+    if (m_visible) {
         invalidate(true);
-    else
-        Compositor::the().invalidate_screen(frame().render_rect());
+    } else {
+        // Since the window won't be visible we need to invalidate the last rendered
+        // rectangles before the next occlusion calculation
+        invalidate_last_rendered_screen_rects_now();
+    }
 }
 
 void Window::set_frameless(bool frameless)
 {
     if (m_frameless == frameless)
         return;
-    auto render_rect_before = frame().render_rect();
     m_frameless = frameless;
     if (m_visible) {
         Compositor::the().invalidate_occlusions();
         invalidate(true, true);
-        Compositor::the().invalidate_screen(frameless ? render_rect_before : frame().render_rect());
+        invalidate_last_rendered_screen_rects();
     }
 }
 
@@ -651,6 +659,24 @@ bool Window::invalidate_no_notify(const Gfx::IntRect& rect, bool with_frame)
     return true;
 }
 
+void Window::invalidate_last_rendered_screen_rects()
+{
+    m_invalidate_last_render_rects = true;
+    Compositor::the().invalidate_occlusions();
+}
+
+void Window::invalidate_last_rendered_screen_rects_now()
+{
+    // We can't wait for the next occlusion computation because the window will either no longer
+    // be around or won't be visible anymore. So we need to invalidate the last rendered rects now.
+    if (!m_opaque_rects.is_empty())
+        Compositor::the().invalidate_screen(m_opaque_rects);
+    if (!m_transparency_rects.is_empty())
+        Compositor::the().invalidate_screen(m_transparency_rects);
+    m_invalidate_last_render_rects = false;
+    Compositor::the().invalidate_occlusions();
+}
+
 void Window::refresh_client_size()
 {
     client()->async_window_resized(m_window_id, m_rect);
@@ -686,11 +712,7 @@ void Window::clear_dirty_rects()
 
 bool Window::is_active() const
 {
-    if (!m_window_stack) {
-        // This may be called while destroying a window as part of
-        // determining what the render rectangle is!
-        return false;
-    }
+    VERIFY(m_window_stack);
     return m_window_stack->active_window() == this;
 }
 
@@ -793,8 +815,7 @@ void Window::handle_window_menu_action(WindowMenuAction action)
         m_should_show_menubar = item.is_checked();
         frame().invalidate();
         recalculate_rect();
-        Compositor::the().invalidate_occlusions();
-        Compositor::the().invalidate_screen();
+        invalidate_last_rendered_screen_rects();
         break;
     }
     }
diff --git a/Userland/Services/WindowServer/Window.h b/Userland/Services/WindowServer/Window.h
index c7eff68385..51aee13c35 100644
--- a/Userland/Services/WindowServer/Window.h
+++ b/Userland/Services/WindowServer/Window.h
@@ -201,6 +201,9 @@ public:
     void invalidate(const Gfx::IntRect&, bool with_frame = false);
     void invalidate_menubar();
     bool invalidate_no_notify(const Gfx::IntRect& rect, bool with_frame = false);
+    void invalidate_last_rendered_screen_rects();
+    void invalidate_last_rendered_screen_rects_now();
+    [[nodiscard]] bool should_invalidate_last_rendered_screen_rects() { return exchange(m_invalidate_last_render_rects, false); }
 
     void refresh_client_size();
 
@@ -409,6 +412,7 @@ private:
     bool m_hit_testing_enabled { true };
     bool m_modified { false };
     bool m_moving_to_another_stack { false };
+    bool m_invalidate_last_render_rects { false };
     WindowTileType m_tiled { WindowTileType::None };
     Gfx::IntRect m_untiled_rect;
     bool m_occluded { false };
diff --git a/Userland/Services/WindowServer/WindowFrame.cpp b/Userland/Services/WindowServer/WindowFrame.cpp
index 15b6ba2e66..8b72964733 100644
--- a/Userland/Services/WindowServer/WindowFrame.cpp
+++ b/Userland/Services/WindowServer/WindowFrame.cpp
@@ -598,25 +598,7 @@ void WindowFrame::window_rect_changed(const Gfx::IntRect& old_rect, const Gfx::I
 {
     layout_buttons();
 
-    auto new_frame_rect = constrained_render_rect_to_screen(frame_rect_for_window(m_window, new_rect));
     set_dirty(true);
-    auto& compositor = Compositor::the();
-
-    {
-        // Invalidate the areas outside of the new rect. Use the last computed occlusions for this purpose
-        // as we can't reliably calculate the previous frame rect anymore. The window state (e.g. maximized
-        // or tiled) may affect the calculations and it may have already been changed by the time we get
-        // called here.
-        auto invalidate_opaque = m_window.opaque_rects().shatter(new_frame_rect);
-        for (auto& rect : invalidate_opaque.rects())
-            compositor.invalidate_screen(rect);
-        auto invalidate_transparent = m_window.transparency_rects().shatter(new_frame_rect);
-        for (auto& rect : invalidate_transparent.rects())
-            compositor.invalidate_screen(rect);
-    }
-
-    compositor.invalidate_occlusions();
-
     WindowManager::the().notify_rect_changed(m_window, old_rect, new_rect);
 }
 
diff --git a/Userland/Services/WindowServer/WindowManager.cpp b/Userland/Services/WindowServer/WindowManager.cpp
index 3b15ac4242..9d548eed80 100644
--- a/Userland/Services/WindowServer/WindowManager.cpp
+++ b/Userland/Services/WindowServer/WindowManager.cpp
@@ -399,7 +399,7 @@ void WindowManager::remove_window(Window& window)
     if (active == &window || active_input == &window || (active && window.is_descendant_of(*active)) || (active_input && active_input != active && window.is_descendant_of(*active_input)))
         pick_new_active_window(&window);
 
-    Compositor::the().invalidate_screen(window.frame().render_rect());
+    window.invalidate_last_rendered_screen_rects_now();
 
     if (m_switcher.is_visible() && window.type() != WindowType::WindowSwitcher)
         m_switcher.refresh();