Kernel: Fix heap expansions deadlock

If a heap expansion is triggered by allocating from e.g. the
RangeAllocator, which may be holding a spin lock, we cannot
immediately allocate another block of backup memory, which could
require the same locks to be acquired. So, defer allocating the
backup memory

Fixes #4675

1 files changed, 12 insertions, 1 deletions

diff --git a/Kernel/Heap/kmalloc.cpp b/Kernel/Heap/kmalloc.cpp
index bb46edc4f4..33210beafd 100644
--- a/Kernel/Heap/kmalloc.cpp
+++ b/Kernel/Heap/kmalloc.cpp
@@ -52,6 +52,8 @@
 
 static RecursiveSpinLock s_lock; // needs to be recursive because of dump_backtrace()
 
+static void kmalloc_allocate_backup_memory();
+
 struct KmallocGlobalHeap {
     struct ExpandGlobalHeap {
         KmallocGlobalHeap& m_global_heap;
@@ -94,7 +96,11 @@ struct KmallocGlobalHeap {
             // backup heap before returning. Otherwise we potentially lose
             // the ability to expand the heap next time we get called.
             ScopeGuard guard([&]() {
-                m_global_heap.allocate_backup_memory();
+                // We may need to defer allocating backup memory because the
+                // heap expansion may have been triggered while holding some
+                // other spinlock. If the expansion happens to need the same
+                // spinlock we would deadlock. So, if we're in any lock, defer
+                Processor::current().deferred_call_queue(kmalloc_allocate_backup_memory);
             });
 
             // Now that we added our backup memory, check if the backup heap
@@ -197,6 +203,11 @@ bool g_dump_kmalloc_stacks;
 static u8* s_next_eternal_ptr;
 static u8* s_end_of_eternal_range;
 
+static void kmalloc_allocate_backup_memory()
+{
+    g_kmalloc_global->allocate_backup_memory();
+}
+
 void kmalloc_enable_expand()
 {
     g_kmalloc_global->allocate_backup_memory();