LibPthread: Implement destruction of pthread_keys

Add a function to destroy any keys that were set on the current thread
using the algorithm from Dr. POSIX's pthread_key_create. Add some
defines to pthread.h for pthread key use, and implement
pthread_key_delete. It has a prototype in pthread.h, but any program
trying to actually use it would be in for a link-time surprise.

Currently, keys are destroyed either via global destructors, with the
s_key_destroyer object, or in exit_thread. exit_thread is invoked by
pthread_exit, and transitively by pthread_create, via the
pthread_create_helper that ensures all threads created with the pthread
API properly clean up for themselves when they exit gracefully.

A future patch might make s_key_destroyer a C++11 thread_local instead,
assuming we get thread_local and thread_local destructors working.

2 files changed, 57 insertions, 5 deletions

diff --git a/Libraries/LibPthread/pthread.cpp b/Libraries/LibPthread/pthread.cpp
index 74fcab0302..9fe543a3de 100644
--- a/Libraries/LibPthread/pthread.cpp
+++ b/Libraries/LibPthread/pthread.cpp
@@ -42,12 +42,23 @@
 
 namespace {
 using PthreadAttrImpl = Syscall::SC_create_thread_params;
+
+struct KeyDestroyer {
+    ~KeyDestroyer() { destroy_for_current_thread(); }
+    static void destroy_for_current_thread();
+};
+
 } // end anonymous namespace
 
 constexpr size_t required_stack_alignment = 4 * MiB;
 constexpr size_t highest_reasonable_guard_size = 32 * PAGE_SIZE;
 constexpr size_t highest_reasonable_stack_size = 8 * MiB; // That's the default in Ubuntu?
 
+// Create an RAII object with a global destructor to destroy pthread keys for the main thread.
+// Impact of this: Any global object that wants to do something with pthread_getspecific
+// in its destructor from the main thread might be in for a nasty surprise.
+static KeyDestroyer s_key_destroyer;
+
 #define __RETURN_PTHREAD_ERROR(rc) \
     return ((rc) < 0 ? -(rc) : 0)
 
@@ -91,6 +102,7 @@ static int create_thread(pthread_t* thread, void* (*entry)(void*), void* argumen
 
 [[noreturn]] static void exit_thread(void* code)
 {
+    KeyDestroyer::destroy_for_current_thread();
     syscall(SC_exit_thread, code);
     ASSERT_NOT_REACHED();
 }
@@ -540,19 +552,18 @@ int pthread_cond_broadcast(pthread_cond_t* cond)
     return 0;
 }
 
-static const int max_keys = 64;
+static constexpr int max_keys = PTHREAD_KEYS_MAX;
 
 typedef void (*KeyDestructor)(void*);
 
 struct KeyTable {
-    // FIXME: Invoke key destructors on thread exit!
-    KeyDestructor destructors[64] { nullptr };
+    KeyDestructor destructors[max_keys] { nullptr };
     int next { 0 };
     pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 };
 
 struct SpecificTable {
-    void* values[64] { nullptr };
+    void* values[max_keys] { nullptr };
 };
 
 static KeyTable s_keys;
@@ -564,7 +575,7 @@ int pthread_key_create(pthread_key_t* key, KeyDestructor destructor)
     int ret = 0;
     pthread_mutex_lock(&s_keys.mutex);
     if (s_keys.next >= max_keys) {
-        ret = ENOMEM;
+        ret = EAGAIN;
     } else {
         *key = s_keys.next++;
         s_keys.destructors[*key] = destructor;
@@ -574,6 +585,16 @@ int pthread_key_create(pthread_key_t* key, KeyDestructor destructor)
     return ret;
 }
 
+int pthread_key_delete(pthread_key_t key)
+{
+    if (key < 0 || key >= max_keys)
+        return EINVAL;
+    pthread_mutex_lock(&s_keys.mutex);
+    s_keys.destructors[key] = nullptr;
+    pthread_mutex_unlock(&s_keys.mutex);
+    return 0;
+}
+
 void* pthread_getspecific(pthread_key_t key)
 {
     if (key < 0)
@@ -593,6 +614,34 @@ int pthread_setspecific(pthread_key_t key, const void* value)
     t_specifics.values[key] = const_cast<void*>(value);
     return 0;
 }
+
+void KeyDestroyer::destroy_for_current_thread()
+{
+    // This function will either be called during exit_thread, for a pthread, or
+    // during global program shutdown for the main thread.
+
+    pthread_mutex_lock(&s_keys.mutex);
+    size_t num_used_keys = s_keys.next;
+
+    // Dr. POSIX accounts for weird key destructors setting their own key again.
+    // Or even, setting other unrelated keys? Odd, but whatever the Doc says goes.
+
+    for (size_t destruct_iteration = 0; destruct_iteration < PTHREAD_DESTRUCTOR_ITERATIONS; ++destruct_iteration) {
+        bool any_nonnull_destructors = false;
+        for (size_t key_index = 0; key_index < num_used_keys; ++key_index) {
+            void* value = exchange(t_specifics.values[key_index], nullptr);
+
+            if (value && s_keys.destructors[key_index]) {
+                any_nonnull_destructors = true;
+                (*s_keys.destructors[key_index])(value);
+            }
+        }
+        if (!any_nonnull_destructors)
+            break;
+    }
+    pthread_mutex_unlock(&s_keys.mutex);
+}
+
 int pthread_setname_np(pthread_t thread, const char* name)
 {
     if (!name)
diff --git a/Libraries/LibPthread/pthread.h b/Libraries/LibPthread/pthread.h
index fb5c4f2dbf..617aac8a90 100644
--- a/Libraries/LibPthread/pthread.h
+++ b/Libraries/LibPthread/pthread.h
@@ -87,6 +87,9 @@ int pthread_setschedparam(pthread_t thread, int policy, const struct sched_param
         0, 0, CLOCK_MONOTONIC_COARSE \
     }
 
+#define PTHREAD_KEYS_MAX 64
+#define PTHREAD_DESTRUCTOR_ITERATIONS 4
+
 int pthread_key_create(pthread_key_t* key, void (*destructor)(void*));
 int pthread_key_delete(pthread_key_t key);
 int pthread_cond_broadcast(pthread_cond_t*);