Kernel: Refactor scheduler to use dynamic thread priorities

Threads now have numeric priorities with a base priority in the 1-99
range.

Whenever a runnable thread is *not* scheduled, its effective priority
is incremented by 1. This is tracked in Thread::m_extra_priority.
The effective priority of a thread is m_priority + m_extra_priority.

When a runnable thread *is* scheduled, its m_extra_priority is reset to
zero and the effective priority returns to base.

This means that lower-priority threads will always eventually get
scheduled to run, once its effective priority becomes high enough to
exceed the base priority of threads "above" it.

The previous values for ThreadPriority (Low, Normal and High) are now
replaced as follows:

    Low -> 10
    Normal -> 30
    High -> 50

In other words, it will take 20 ticks for a "Low" priority thread to
get to "Normal" effective priority, and another 20 to reach "High".

This is not perfect, and I've used some quite naive data structures,
but I think the mechanism will allow us to build various new and
interesting optimizations, and we can figure out better data structures
later on. :^)

8 files changed, 77 insertions, 116 deletions

diff --git a/Kernel/FileSystem/ProcFS.cpp b/Kernel/FileSystem/ProcFS.cpp
index a3ea6c78ae..a5ec55f01a 100644
--- a/Kernel/FileSystem/ProcFS.cpp
+++ b/Kernel/FileSystem/ProcFS.cpp
@@ -767,7 +767,8 @@ Optional<KBuffer> procfs$all(InodeIdentifier)
             thread_object.add("times_scheduled", thread.times_scheduled());
             thread_object.add("ticks", thread.ticks());
             thread_object.add("state", thread.state_string());
-            thread_object.add("priority", to_string(thread.priority()));
+            thread_object.add("priority", thread.priority());
+            thread_object.add("effective_priority", thread.effective_priority());
             thread_object.add("syscall_count", thread.syscall_count());
             thread_object.add("inode_faults", thread.inode_faults());
             thread_object.add("zero_faults", thread.zero_faults());
diff --git a/Kernel/Process.cpp b/Kernel/Process.cpp
index 8ec2c13f25..0248557d02 100644
--- a/Kernel/Process.cpp
+++ b/Kernel/Process.cpp
@@ -2831,10 +2831,10 @@ int Process::sys$sched_setparam(pid_t pid, const struct sched_param* param)
     if (!is_superuser() && m_euid != peer->m_uid && m_uid != peer->m_uid)
         return -EPERM;
 
-    if (param->sched_priority < (int)ThreadPriority::First || param->sched_priority > (int)ThreadPriority::Last)
+    if (param->sched_priority < THREAD_PRIORITY_MIN || param->sched_priority > THREAD_PRIORITY_MAX)
         return -EINVAL;
 
-    peer->any_thread().set_priority((ThreadPriority)param->sched_priority);
+    peer->any_thread().set_priority((u32)param->sched_priority);
     return 0;
 }
 
@@ -3078,13 +3078,10 @@ int Process::sys$create_thread(void* (*entry)(void*), void* argument, const Sysc
 
     // FIXME: return EAGAIN if Thread::all_threads().size() is greater than PTHREAD_THREADS_MAX
 
-    ThreadPriority requested_thread_priority = static_cast<ThreadPriority>(params->m_schedule_priority);
-    if (requested_thread_priority < ThreadPriority::First || requested_thread_priority > ThreadPriority::Last)
+    int requested_thread_priority = params->m_schedule_priority;
+    if (requested_thread_priority < THREAD_PRIORITY_MIN || requested_thread_priority > THREAD_PRIORITY_MAX)
         return -EINVAL;
 
-    if (requested_thread_priority != ThreadPriority::Normal && !is_superuser())
-        return -EPERM;
-
     bool is_thread_joinable = (0 == params->m_detach_state);
 
     // FIXME: Do something with guard pages?
diff --git a/Kernel/Process.h b/Kernel/Process.h
index eff6a46674..df904b5f25 100644
--- a/Kernel/Process.h
+++ b/Kernel/Process.h
@@ -432,8 +432,6 @@ private:
     Process& m_process;
 };
 
-const char* to_string(ThreadPriority);
-
 extern InlineLinkedList<Process>* g_processes;
 
 template<typename Callback>
diff --git a/Kernel/Scheduler.cpp b/Kernel/Scheduler.cpp
index febb8a1f19..0f270dd2ff 100644
--- a/Kernel/Scheduler.cpp
+++ b/Kernel/Scheduler.cpp
@@ -1,3 +1,4 @@
+#include <AK/QuickSort.h>
 #include <AK/TemporaryChange.h>
 #include <Kernel/Arch/i386/PIT.h>
 #include <Kernel/FileSystem/FileDescription.h>
@@ -21,7 +22,7 @@ void Scheduler::init_thread(Thread& thread)
 void Scheduler::update_state_for_thread(Thread& thread)
 {
     ASSERT_INTERRUPTS_DISABLED();
-    auto& list = g_scheduler_data->thread_list_for_state_and_priority(thread.state(), thread.priority());
+    auto& list = g_scheduler_data->thread_list_for_state(thread.state());
 
     if (list.contains(thread))
         return;
@@ -29,35 +30,12 @@ void Scheduler::update_state_for_thread(Thread& thread)
     list.append(thread);
 }
 
-template<typename Callback>
-static inline IterationDecision for_each_runnable_with_priority(ThreadPriority priority, Callback callback)
-{
-    ASSERT_INTERRUPTS_DISABLED();
-    auto& tl = g_scheduler_data->m_runnable_threads[(u8)priority - (u8)ThreadPriority::First];
-    for (auto it = tl.begin(); it != tl.end();) {
-        auto& thread = *it;
-        it = ++it;
-        if (callback(thread) == IterationDecision::Break)
-            return IterationDecision::Break;
-    }
-
-    return IterationDecision::Continue;
-}
-
-static u32 time_slice_for(ThreadPriority priority)
+static u32 time_slice_for(const Thread& thread)
 {
     // One time slice unit == 1ms
-    switch (priority) {
-    case ThreadPriority::High:
-        return 20;
-    case ThreadPriority::Normal:
-        return 15;
-    case ThreadPriority::Low:
-        return 5;
-    case ThreadPriority::Idle:
+    if (&thread == g_colonel)
         return 1;
-    }
-    ASSERT_NOT_REACHED();
+    return 10;
 }
 
 Thread* current;
@@ -364,46 +342,54 @@ bool Scheduler::pick_next()
 #ifdef SCHEDULER_RUNNABLE_DEBUG
     dbgprintf("Non-runnables:\n");
     Scheduler::for_each_nonrunnable([](Thread& thread) -> IterationDecision {
-        auto& process = thread.process();
-        dbgprintf("[K%x] %-12s %s(%u:%u) @ %w:%x\n", &process, thread.state_string(), process.name().characters(), process.pid(), thread.tid(), thread.tss().cs, thread.tss().eip);
+        dbgprintf("  %-12s %s(%u:%u) @ %w:%x\n", thread.state_string(), thread.name().characters(), thread.pid(), thread.tid(), thread.tss().cs, thread.tss().eip);
         return IterationDecision::Continue;
     });
 
-    for (u8 priority = (u8)ThreadPriority::Last; priority >= (u8)ThreadPriority::First; --priority) {
-        dbgprintf("Runnables (%s):\n", to_string((ThreadPriority)priority));
-        for_each_runnable_with_priority((ThreadPriority)priority, [](Thread& thread) -> IterationDecision {
-            auto& process = thread.process();
-            dbgprintf("[K%x] %-12s %s(%u:%u) @ %w:%x\n", &process, thread.state_string(), process.name().characters(), process.pid(), thread.tid(), thread.tss().cs, thread.tss().eip);
-            return IterationDecision::Continue;
-        });
-    }
+    dbgprintf("Runnables:\n");
+    Scheduler::for_each_runnable([](Thread& thread) -> IterationDecision {
+        dbgprintf("  %3u/%2u %-12s %s(%u:%u) @ %w:%x\n", thread.effective_priority(), thread.priority(), thread.state_string(), thread.name().characters(), thread.pid(), thread.tid(), thread.tss().cs, thread.tss().eip);
+        return IterationDecision::Continue;
+    });
 #endif
 
-    for (u8 priority = (u8)ThreadPriority::Last; priority >= (u8)ThreadPriority::First; --priority) {
-        auto& runnable_list = g_scheduler_data->m_runnable_threads[priority - (u8)ThreadPriority::First];
-        if (runnable_list.is_empty())
-            continue;
+    Vector<Thread*, 128> sorted_runnables;
+    for_each_runnable([&sorted_runnables](auto& thread) {
+        sorted_runnables.append(&thread);
+        return IterationDecision::Continue;
+    });
+    quick_sort(sorted_runnables.begin(), sorted_runnables.end(), [](auto& a, auto& b) { return a->effective_priority() >= b->effective_priority(); });
 
-        auto* previous_head = runnable_list.first();
-        for (;;) {
-            // Move head to tail.
-            runnable_list.append(*runnable_list.first());
-            auto* thread = runnable_list.first();
+    Thread* thread_to_schedule = nullptr;
 
-            if (!thread->process().is_being_inspected() && (thread->state() == Thread::Runnable || thread->state() == Thread::Running)) {
-#ifdef SCHEDULER_DEBUG
-                dbgprintf("switch to %s(%u:%u) @ %w:%x\n", thread->process().name().characters(), thread->process().pid(), thread->tid(), thread->tss().cs, thread->tss().eip);
-#endif
-                return context_switch(*thread);
-            }
+    for (auto* thread : sorted_runnables) {
+        if (thread->process().is_being_inspected())
+            continue;
 
-            if (thread == previous_head)
-                break;
+        ASSERT(thread->state() == Thread::Runnable || thread->state() == Thread::Running);
+
+        if (!thread_to_schedule) {
+            thread->m_extra_priority = 0;
+            thread_to_schedule = thread;
+        } else {
+            thread->m_extra_priority++;
         }
     }
 
-    // Nothing wants to run. Send in the colonel!
-    return context_switch(*g_colonel);
+
+    if (!thread_to_schedule)
+        thread_to_schedule = g_colonel;
+
+#ifdef SCHEDULER_DEBUG
+    dbgprintf("switch to %s(%u:%u) @ %w:%x\n",
+        thread_to_schedule->name().characters(),
+        thread_to_schedule->pid(),
+        thread_to_schedule->tid(),
+        thread_to_schedule->tss().cs,
+        thread_to_schedule->tss().eip);
+#endif
+
+    return context_switch(*thread_to_schedule);
 }
 
 bool Scheduler::donate_to(Thread* beneficiary, const char* reason)
@@ -417,7 +403,7 @@ bool Scheduler::donate_to(Thread* beneficiary, const char* reason)
     if (!beneficiary || beneficiary->state() != Thread::Runnable || ticks_left <= 1)
         return yield();
 
-    unsigned ticks_to_donate = min(ticks_left - 1, time_slice_for(beneficiary->priority()));
+    unsigned ticks_to_donate = min(ticks_left - 1, time_slice_for(*beneficiary));
 #ifdef SCHEDULER_DEBUG
     dbgprintf("%s(%u:%u) donating %u ticks to %s(%u:%u), reason=%s\n", current->process().name().characters(), current->pid(), current->tid(), ticks_to_donate, beneficiary->process().name().characters(), beneficiary->pid(), beneficiary->tid(), reason);
 #endif
@@ -459,7 +445,7 @@ void Scheduler::switch_now()
 
 bool Scheduler::context_switch(Thread& thread)
 {
-    thread.set_ticks_left(time_slice_for(thread.priority()));
+    thread.set_ticks_left(time_slice_for(thread));
     thread.did_schedule();
 
     if (current == &thread)
@@ -472,10 +458,10 @@ bool Scheduler::context_switch(Thread& thread)
             current->set_state(Thread::Runnable);
 
 #ifdef LOG_EVERY_CONTEXT_SWITCH
-        dbgprintf("Scheduler: %s(%u:%u) -> %s(%u:%u) [%s] %w:%x\n",
+        dbgprintf("Scheduler: %s(%u:%u) -> %s(%u:%u) [%u] %w:%x\n",
             current->process().name().characters(), current->process().pid(), current->tid(),
             thread.process().name().characters(), thread.process().pid(), thread.tid(),
-            to_string(thread.priority()),
+            thread.priority(),
             thread.tss().cs, thread.tss().eip);
 #endif
     }
@@ -552,8 +538,7 @@ void Scheduler::initialize()
     s_redirection.selector = gdt_alloc_entry();
     initialize_redirection();
     s_colonel_process = Process::create_kernel_process(g_colonel, "colonel", nullptr);
-    // Make sure the colonel uses a smallish time slice.
-    g_colonel->set_priority(ThreadPriority::Idle);
+    g_colonel->set_priority(THREAD_PRIORITY_MIN);
     load_task_register(s_redirection.selector);
 }
 
diff --git a/Kernel/Syscall.h b/Kernel/Syscall.h
index 3487f264d5..3b5df184c4 100644
--- a/Kernel/Syscall.h
+++ b/Kernel/Syscall.h
@@ -270,7 +270,7 @@ struct SC_futex_params {
 
 struct SC_create_thread_params {
     unsigned int m_detach_state = 0; // JOINABLE or DETACHED
-    int m_schedule_priority = 2;     // ThreadPriority::Normal
+    int m_schedule_priority = 30;    // THREAD_PRIORITY_NORMAL
     // FIXME: Implment guard pages in create_thread (unreadable pages at "overflow" end of stack)
     // "If an implementation rounds up the value of guardsize to a multiple of {PAGESIZE},
     // a call to pthread_attr_getguardsize() specifying attr shall store in the guardsize
diff --git a/Kernel/Thread.cpp b/Kernel/Thread.cpp
index 9201752453..92078cae45 100644
--- a/Kernel/Thread.cpp
+++ b/Kernel/Thread.cpp
@@ -765,23 +765,6 @@ const LogStream& operator<<(const LogStream& stream, const Thread& value)
     return stream << value.process().name() << "(" << value.pid() << ":" << value.tid() << ")";
 }
 
-const char* to_string(ThreadPriority priority)
-{
-    switch (priority) {
-    case ThreadPriority::Idle:
-        return "Idle";
-    case ThreadPriority::Low:
-        return "Low";
-    case ThreadPriority::Normal:
-        return "Normal";
-    case ThreadPriority::High:
-        return "High";
-    }
-    dbg() << "to_string(ThreadPriority): Invalid priority: " << (u32)priority;
-    ASSERT_NOT_REACHED();
-    return nullptr;
-}
-
 void Thread::wait_on(WaitQueue& queue, Thread* beneficiary, const char* reason)
 {
     bool did_unlock = unlock_process_if_locked();
diff --git a/Kernel/Thread.h b/Kernel/Thread.h
index 4ce7a86bcd..63e6941e81 100644
--- a/Kernel/Thread.h
+++ b/Kernel/Thread.h
@@ -35,14 +35,11 @@ struct ThreadSpecificData {
     ThreadSpecificData* self;
 };
 
-enum class ThreadPriority : u8 {
-    Idle,
-    Low,
-    Normal,
-    High,
-    First = Low,
-    Last = High,
-};
+#define THREAD_PRIORITY_MIN 1
+#define THREAD_PRIORITY_LOW 10
+#define THREAD_PRIORITY_NORMAL 30
+#define THREAD_PRIORITY_HIGH 50
+#define THREAD_PRIORITY_MAX 99
 
 class Thread {
     friend class Process;
@@ -61,8 +58,10 @@ public:
     int tid() const { return m_tid; }
     int pid() const;
 
-    void set_priority(ThreadPriority p) { m_priority = p; }
-    ThreadPriority priority() const { return m_priority; }
+    void set_priority(u32 p) { m_priority = p; }
+    u32 priority() const { return m_priority; }
+
+    u32 effective_priority() const { return m_priority + m_extra_priority; }
 
     void set_joinable(bool j) { m_is_joinable = j; }
     bool is_joinable() const { return m_is_joinable; }
@@ -451,7 +450,8 @@ private:
     FPUState* m_fpu_state { nullptr };
     State m_state { Invalid };
     String m_name;
-    ThreadPriority m_priority { ThreadPriority::Normal };
+    u32 m_priority { THREAD_PRIORITY_NORMAL };
+    u32 m_extra_priority { 0 };
     bool m_has_used_fpu { false };
     bool m_dump_backtrace_on_finalization { false };
     bool m_should_die { false };
@@ -501,15 +501,13 @@ const LogStream& operator<<(const LogStream&, const Thread&);
 struct SchedulerData {
     typedef IntrusiveList<Thread, &Thread::m_runnable_list_node> ThreadList;
 
-    static constexpr size_t num_thread_priorities = (size_t)ThreadPriority::Last - (size_t)ThreadPriority::First + 1;
-    ThreadList m_runnable_threads[num_thread_priorities];
-
+    ThreadList m_runnable_threads;
     ThreadList m_nonrunnable_threads;
 
-    ThreadList& thread_list_for_state_and_priority(Thread::State state, ThreadPriority priority)
+    ThreadList& thread_list_for_state(Thread::State state)
     {
         if (Thread::is_runnable_state(state))
-            return m_runnable_threads[(u8)priority - (u8)ThreadPriority::First];
+            return m_runnable_threads;
         return m_nonrunnable_threads;
     }
 };
@@ -518,13 +516,12 @@ template<typename Callback>
 inline IterationDecision Scheduler::for_each_runnable(Callback callback)
 {
     ASSERT_INTERRUPTS_DISABLED();
-    for (auto& tl : g_scheduler_data->m_runnable_threads) {
-        for (auto it = tl.begin(); it != tl.end();) {
-            auto& thread = *it;
-            it = ++it;
-            if (callback(thread) == IterationDecision::Break)
-                return IterationDecision::Break;
-        }
+    auto& tl = g_scheduler_data->m_runnable_threads;
+    for (auto it = tl.begin(); it != tl.end();) {
+        auto& thread = *it;
+        it = ++it;
+        if (callback(thread) == IterationDecision::Break)
+            return IterationDecision::Break;
     }
 
     return IterationDecision::Continue;
diff --git a/Kernel/init.cpp b/Kernel/init.cpp
index f85640d9b7..8acf7460fc 100644
--- a/Kernel/init.cpp
+++ b/Kernel/init.cpp
@@ -175,7 +175,7 @@ VFS* vfs;
             kprintf("init_stage2: error spawning Shell: %d\n", error);
             hang();
         }
-        thread->set_priority(ThreadPriority::High);
+        thread->set_priority(THREAD_PRIORITY_HIGH);
     } else {
         tty0->set_graphical(true);
         Thread* thread = nullptr;
@@ -184,7 +184,7 @@ VFS* vfs;
             kprintf("init_stage2: error spawning SystemServer: %d\n", error);
             hang();
         }
-        thread->set_priority(ThreadPriority::High);
+        thread->set_priority(THREAD_PRIORITY_HIGH);
     }
     {
         Thread* thread = nullptr;
@@ -326,7 +326,7 @@ extern "C" [[noreturn]] void init(u32 physical_address_for_kernel_page_tables)
     });
 
     Process::create_kernel_process(g_finalizer, "Finalizer", [] {
-        current->set_priority(ThreadPriority::Low);
+        current->set_priority(THREAD_PRIORITY_LOW);
         for (;;) {
             current->wait_on(*g_finalizer_wait_queue);
             Thread::finalize_dying_threads();