LibC: Don't format strings when asserting with an unstable heap

If we hit an assertion while the heap isn't in a stable state, we can't
rely on dynamic memory allocation because the malloc mutex is already
held and the heap is most likely corrupted. Instead, we need to bail
out fast before we make the situation even worse.

3 files changed, 13 insertions, 4 deletions

diff --git a/Userland/Libraries/LibC/assert.cpp b/Userland/Libraries/LibC/assert.cpp
index f17f59d1ce..c3a0961835 100644
--- a/Userland/Libraries/LibC/assert.cpp
+++ b/Userland/Libraries/LibC/assert.cpp
@@ -19,9 +19,11 @@ extern bool __stdio_is_initialized;
 #ifndef NDEBUG
 void __assertion_failed(const char* msg)
 {
-    dbgln("ASSERTION FAILED: {}", msg);
-    if (__stdio_is_initialized)
-        warnln("ASSERTION FAILED: {}", msg);
+    if (__heap_is_stable) {
+        dbgln("ASSERTION FAILED: {}", msg);
+        if (__stdio_is_initialized)
+            warnln("ASSERTION FAILED: {}", msg);
+    }
 
     Syscall::SC_set_coredump_metadata_params params {
         { "assertion", strlen("assertion") },
diff --git a/Userland/Libraries/LibC/malloc.cpp b/Userland/Libraries/LibC/malloc.cpp
index 502025ad38..8819f020db 100644
--- a/Userland/Libraries/LibC/malloc.cpp
+++ b/Userland/Libraries/LibC/malloc.cpp
@@ -26,8 +26,13 @@ public:
         : m_mutex(mutex)
     {
         lock();
+        __heap_is_stable = false;
+    }
+    ALWAYS_INLINE ~PthreadMutexLocker()
+    {
+        __heap_is_stable = true;
+        unlock();
     }
-    ALWAYS_INLINE ~PthreadMutexLocker() { unlock(); }
     ALWAYS_INLINE void lock() { pthread_mutex_lock(&m_mutex); }
     ALWAYS_INLINE void unlock() { pthread_mutex_unlock(&m_mutex); }
 
@@ -38,6 +43,7 @@ private:
 #define RECYCLE_BIG_ALLOCATIONS
 
 static pthread_mutex_t s_malloc_mutex = PTHREAD_MUTEX_INITIALIZER;
+bool __heap_is_stable = true;
 
 constexpr size_t number_of_hot_chunked_blocks_to_keep_around = 16;
 constexpr size_t number_of_cold_chunked_blocks_to_keep_around = 16;
diff --git a/Userland/Libraries/LibC/sys/internals.h b/Userland/Libraries/LibC/sys/internals.h
index 1ddebc0eb3..9f637eaf3f 100644
--- a/Userland/Libraries/LibC/sys/internals.h
+++ b/Userland/Libraries/LibC/sys/internals.h
@@ -18,6 +18,7 @@ extern void __stdio_init();
 extern void _init();
 extern bool __environ_is_malloced;
 extern bool __stdio_is_initialized;
+extern bool __heap_is_stable;
 
 int __cxa_atexit(AtExitFunction exit_function, void* parameter, void* dso_handle);
 void __cxa_finalize(void* dso_handle);