diff options
Diffstat (limited to 'Kernel/Arch/x86/CPU.h')
| -rw-r--r-- | Kernel/Arch/x86/CPU.h | 1063 |
1 files changed, 1063 insertions, 0 deletions
diff --git a/Kernel/Arch/x86/CPU.h b/Kernel/Arch/x86/CPU.h new file mode 100644 index 0000000000..34c5b43fa3 --- /dev/null +++ b/Kernel/Arch/x86/CPU.h @@ -0,0 +1,1063 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include <AK/Atomic.h> +#include <AK/Badge.h> +#include <AK/Noncopyable.h> +#include <AK/Vector.h> + +#include <Kernel/Arch/x86/DescriptorTable.h> +#include <Kernel/Arch/x86/TSS.h> + +#include <Kernel/PhysicalAddress.h> +#include <Kernel/VirtualAddress.h> +#include <LibC/sys/arch/i386/regs.h> + +#define READONLY_AFTER_INIT __attribute__((section(".ro_after_init"))) +#define UNMAP_AFTER_INIT NEVER_INLINE __attribute__((section(".unmap_after_init"))) + +#define PAGE_SIZE 4096 +#define GENERIC_INTERRUPT_HANDLERS_COUNT (256 - IRQ_VECTOR_BASE) +#define PAGE_MASK ((FlatPtr)0xfffff000u) + +namespace Kernel { + +class MemoryManager; +class PageDirectory; +class PageTableEntry; + +static constexpr u32 safe_eflags_mask = 0xdff; +static constexpr u32 iopl_mask = 3u << 12; + +inline u32 get_iopl_from_eflags(u32 eflags) +{ + return (eflags & iopl_mask) >> 12; +} + +class PageDirectoryEntry { +public: + const PageTableEntry* page_table_base() const { return reinterpret_cast<PageTableEntry*>(m_raw & 0xfffff000u); } + PageTableEntry* page_table_base() { return reinterpret_cast<PageTableEntry*>(m_raw & 0xfffff000u); } + void set_page_table_base(u32 value) + { + m_raw &= 0x8000000000000fffULL; + m_raw |= value & 0xfffff000; + } + + bool is_null() const { return m_raw == 0; } + void clear() { m_raw = 0; } + + u64 raw() const { return m_raw; } + void copy_from(Badge<PageDirectory>, const PageDirectoryEntry& other) { m_raw = other.m_raw; } + + enum Flags { + Present = 1 << 0, + ReadWrite = 1 << 1, + UserSupervisor = 1 << 2, + WriteThrough = 1 << 3, + CacheDisabled = 1 << 4, + Huge = 1 << 7, + Global = 1 << 8, + NoExecute = 0x8000000000000000ULL, + }; + + bool is_present() const { return raw() & Present; } + void set_present(bool b) { set_bit(Present, b); } + + bool is_user_allowed() const { return raw() & UserSupervisor; } + void set_user_allowed(bool b) { set_bit(UserSupervisor, b); } + + bool is_huge() const { return raw() & Huge; } + void set_huge(bool b) { set_bit(Huge, b); } + + bool is_writable() const { return raw() & ReadWrite; } + void set_writable(bool b) { set_bit(ReadWrite, b); } + + bool is_write_through() const { return raw() & WriteThrough; } + void set_write_through(bool b) { set_bit(WriteThrough, b); } + + bool is_cache_disabled() const { return raw() & CacheDisabled; } + void set_cache_disabled(bool b) { set_bit(CacheDisabled, b); } + + bool is_global() const { return raw() & Global; } + void set_global(bool b) { set_bit(Global, b); } + + bool is_execute_disabled() const { return raw() & NoExecute; } + void set_execute_disabled(bool b) { set_bit(NoExecute, b); } + + void set_bit(u64 bit, bool value) + { + if (value) + m_raw |= bit; + else + m_raw &= ~bit; + } + +private: + u64 m_raw; +}; + +class PageTableEntry { +public: + void* physical_page_base() { return reinterpret_cast<void*>(m_raw & 0xfffff000u); } + void set_physical_page_base(u32 value) + { + m_raw &= 0x8000000000000fffULL; + m_raw |= value & 0xfffff000; + } + + u64 raw() const { return (u32)m_raw; } + + enum Flags { + Present = 1 << 0, + ReadWrite = 1 << 1, + UserSupervisor = 1 << 2, + WriteThrough = 1 << 3, + CacheDisabled = 1 << 4, + Global = 1 << 8, + NoExecute = 0x8000000000000000ULL, + }; + + bool is_present() const { return raw() & Present; } + void set_present(bool b) { set_bit(Present, b); } + + bool is_user_allowed() const { return raw() & UserSupervisor; } + void set_user_allowed(bool b) { set_bit(UserSupervisor, b); } + + bool is_writable() const { return raw() & ReadWrite; } + void set_writable(bool b) { set_bit(ReadWrite, b); } + + bool is_write_through() const { return raw() & WriteThrough; } + void set_write_through(bool b) { set_bit(WriteThrough, b); } + + bool is_cache_disabled() const { return raw() & CacheDisabled; } + void set_cache_disabled(bool b) { set_bit(CacheDisabled, b); } + + bool is_global() const { return raw() & Global; } + void set_global(bool b) { set_bit(Global, b); } + + bool is_execute_disabled() const { return raw() & NoExecute; } + void set_execute_disabled(bool b) { set_bit(NoExecute, b); } + + bool is_null() const { return m_raw == 0; } + void clear() { m_raw = 0; } + + void set_bit(u64 bit, bool value) + { + if (value) + m_raw |= bit; + else + m_raw &= ~bit; + } + +private: + u64 m_raw; +}; + +static_assert(sizeof(PageDirectoryEntry) == 8); +static_assert(sizeof(PageTableEntry) == 8); + +class PageDirectoryPointerTable { +public: + PageDirectoryEntry* directory(size_t index) + { + return (PageDirectoryEntry*)(raw[index] & ~0xfffu); + } + + u64 raw[4]; +}; + +class GenericInterruptHandler; +struct RegisterState; + +const DescriptorTablePointer& get_gdtr(); +const DescriptorTablePointer& get_idtr(); +void register_interrupt_handler(u8 number, void (*handler)()); +void register_user_callable_interrupt_handler(u8 number, void (*handler)()); +GenericInterruptHandler& get_interrupt_handler(u8 interrupt_number); +void register_generic_interrupt_handler(u8 number, GenericInterruptHandler&); +void unregister_generic_interrupt_handler(u8 number, GenericInterruptHandler&); +void flush_idt(); +void load_task_register(u16 selector); +[[noreturn]] void handle_crash(RegisterState&, const char* description, int signal, bool out_of_memory = false); + +#define LSW(x) ((u32)(x)&0xFFFF) +#define MSW(x) (((u32)(x) >> 16) & 0xFFFF) +#define LSB(x) ((x)&0xFF) +#define MSB(x) (((x) >> 8) & 0xFF) + +#define cli() asm volatile("cli" :: \ + : "memory") +#define sti() asm volatile("sti" :: \ + : "memory") + +inline FlatPtr cpu_flags() +{ + FlatPtr flags; + asm volatile( + "pushf\n" + "pop %0\n" + : "=rm"(flags)::"memory"); + return flags; +} + +inline void set_fs(u16 segment) +{ + asm volatile( + "mov %%ax, %%fs" ::"a"(segment) + : "memory"); +} + +inline void set_gs(u16 segment) +{ + asm volatile( + "mov %%ax, %%gs" ::"a"(segment) + : "memory"); +} + +inline u16 get_fs() +{ + u16 fs; + asm("mov %%fs, %%eax" + : "=a"(fs)); + return fs; +} + +inline u16 get_gs() +{ + u16 gs; + asm("mov %%gs, %%eax" + : "=a"(gs)); + return gs; +} + +inline u32 read_fs_u32(u32 offset) +{ + u32 val; + asm volatile( + "movl %%fs:%a[off], %k[val]" + : [val] "=r"(val) + : [off] "ir"(offset)); + return val; +} + +inline FlatPtr read_fs_ptr(u32 offset) +{ + return read_fs_u32(offset); +} + +inline void write_fs_u32(u32 offset, u32 val) +{ + asm volatile( + "movl %k[val], %%fs:%a[off]" ::[off] "ir"(offset), [val] "ir"(val) + : "memory"); +} + +inline bool are_interrupts_enabled() +{ + return cpu_flags() & 0x200; +} + +class InterruptDisabler { +public: + InterruptDisabler() + { + m_flags = cpu_flags(); + cli(); + } + + ~InterruptDisabler() + { + if (m_flags & 0x200) + sti(); + } + +private: + u32 m_flags; +}; + +class NonMaskableInterruptDisabler { +public: + NonMaskableInterruptDisabler(); + ~NonMaskableInterruptDisabler(); +}; + +/* Map IRQ0-15 @ ISR 0x50-0x5F */ +#define IRQ_VECTOR_BASE 0x50 + +struct PageFaultFlags { + enum Flags { + NotPresent = 0x00, + ProtectionViolation = 0x01, + Read = 0x00, + Write = 0x02, + UserMode = 0x04, + SupervisorMode = 0x00, + ReservedBitViolation = 0x08, + InstructionFetch = 0x10, + }; +}; + +class PageFault { +public: + PageFault(u16 code, VirtualAddress vaddr) + : m_code(code) + , m_vaddr(vaddr) + { + } + + enum class Type { + PageNotPresent = PageFaultFlags::NotPresent, + ProtectionViolation = PageFaultFlags::ProtectionViolation, + }; + + enum class Access { + Read = PageFaultFlags::Read, + Write = PageFaultFlags::Write, + }; + + VirtualAddress vaddr() const { return m_vaddr; } + u16 code() const { return m_code; } + + Type type() const { return (Type)(m_code & 1); } + Access access() const { return (Access)(m_code & 2); } + + bool is_not_present() const { return (m_code & 1) == PageFaultFlags::NotPresent; } + bool is_protection_violation() const { return (m_code & 1) == PageFaultFlags::ProtectionViolation; } + bool is_read() const { return (m_code & 2) == PageFaultFlags::Read; } + bool is_write() const { return (m_code & 2) == PageFaultFlags::Write; } + bool is_user() const { return (m_code & 4) == PageFaultFlags::UserMode; } + bool is_supervisor() const { return (m_code & 4) == PageFaultFlags::SupervisorMode; } + bool is_instruction_fetch() const { return (m_code & 16) == PageFaultFlags::InstructionFetch; } + +private: + u16 m_code; + VirtualAddress m_vaddr; +}; + +struct [[gnu::packed]] RegisterState { + FlatPtr ss; + FlatPtr gs; + FlatPtr fs; + FlatPtr es; + FlatPtr ds; + FlatPtr edi; + FlatPtr esi; + FlatPtr ebp; + FlatPtr esp; + FlatPtr ebx; + FlatPtr edx; + FlatPtr ecx; + FlatPtr eax; + u16 exception_code; + u16 isr_number; +#if ARCH(X86_64) + u32 padding; +#endif + FlatPtr eip; + FlatPtr cs; + FlatPtr eflags; + FlatPtr userspace_esp; + FlatPtr userspace_ss; +}; + +#if ARCH(I386) +# define REGISTER_STATE_SIZE (19 * 4) +#else +# define REGISTER_STATE_SIZE (19 * 8) +#endif +static_assert(REGISTER_STATE_SIZE == sizeof(RegisterState)); + +void copy_kernel_registers_into_ptrace_registers(PtraceRegisters&, const RegisterState&); +void copy_ptrace_registers_into_kernel_registers(RegisterState&, const PtraceRegisters&); + +struct [[gnu::aligned(16)]] FPUState +{ + u8 buffer[512]; +}; + +constexpr FlatPtr page_base_of(FlatPtr address) +{ + return address & PAGE_MASK; +} + +inline FlatPtr page_base_of(const void* address) +{ + return page_base_of((FlatPtr)address); +} + +constexpr FlatPtr offset_in_page(FlatPtr address) +{ + return address & (~PAGE_MASK); +} + +inline FlatPtr offset_in_page(const void* address) +{ + return offset_in_page((FlatPtr)address); +} + +FlatPtr read_cr0(); +FlatPtr read_cr2(); +FlatPtr read_cr3(); +FlatPtr read_cr4(); +u64 read_xcr0(); + +void write_cr0(FlatPtr); +void write_cr3(FlatPtr); +void write_cr4(FlatPtr); +void write_xcr0(u64); + +FlatPtr read_dr6(); + +static inline bool is_kernel_mode() +{ + u16 cs; + asm volatile( + "mov %%cs, %[cs] \n" + : [cs] "=g"(cs)); + return (cs & 3) == 0; +} + +class CPUID { +public: + explicit CPUID(u32 function) { asm volatile("cpuid" + : "=a"(m_eax), "=b"(m_ebx), "=c"(m_ecx), "=d"(m_edx) + : "a"(function), "c"(0)); } + u32 eax() const { return m_eax; } + u32 ebx() const { return m_ebx; } + u32 ecx() const { return m_ecx; } + u32 edx() const { return m_edx; } + +private: + u32 m_eax { 0xffffffff }; + u32 m_ebx { 0xffffffff }; + u32 m_ecx { 0xffffffff }; + u32 m_edx { 0xffffffff }; +}; + +inline void read_tsc(u32& lsw, u32& msw) +{ + asm volatile("rdtsc" + : "=d"(msw), "=a"(lsw)); +} + +inline u64 read_tsc() +{ + u32 lsw; + u32 msw; + read_tsc(lsw, msw); + return ((u64)msw << 32) | lsw; +} + +// FIXME: This can't hold every CPU feature as-is. +enum class CPUFeature : u32 { + NX = (1 << 0), + PAE = (1 << 1), + PGE = (1 << 2), + RDRAND = (1 << 3), + RDSEED = (1 << 4), + SMAP = (1 << 5), + SMEP = (1 << 6), + SSE = (1 << 7), + TSC = (1 << 8), + RDTSCP = (1 << 9), + CONSTANT_TSC = (1 << 10), + NONSTOP_TSC = (1 << 11), + UMIP = (1 << 12), + SEP = (1 << 13), + SYSCALL = (1 << 14), + MMX = (1 << 15), + SSE2 = (1 << 16), + SSE3 = (1 << 17), + SSSE3 = (1 << 18), + SSE4_1 = (1 << 19), + SSE4_2 = (1 << 20), + XSAVE = (1 << 21), + AVX = (1 << 22), +}; + +class Thread; +struct TrapFrame; + +#define GDT_SELECTOR_CODE0 0x08 +#define GDT_SELECTOR_DATA0 0x10 +#define GDT_SELECTOR_CODE3 0x18 +#define GDT_SELECTOR_DATA3 0x20 +#define GDT_SELECTOR_TLS 0x28 +#define GDT_SELECTOR_PROC 0x30 +#define GDT_SELECTOR_TSS 0x38 + +// SYSENTER makes certain assumptions on how the GDT is structured: +static_assert(GDT_SELECTOR_CODE0 + 8 == GDT_SELECTOR_DATA0); // SS0 = CS0 + 8 + +// SYSEXIT makes certain assumptions on how the GDT is structured: +static_assert(GDT_SELECTOR_CODE0 + 16 == GDT_SELECTOR_CODE3); // CS3 = CS0 + 16 +static_assert(GDT_SELECTOR_CODE0 + 24 == GDT_SELECTOR_DATA3); // SS3 = CS0 + 32 + +class ProcessorInfo; +class SchedulerPerProcessorData; +struct MemoryManagerData; +struct ProcessorMessageEntry; + +struct ProcessorMessage { + enum Type { + FlushTlb, + Callback, + CallbackWithData + }; + Type type; + volatile u32 refs; // atomic + union { + ProcessorMessage* next; // only valid while in the pool + struct { + void (*handler)(); + } callback; + struct { + void* data; + void (*handler)(void*); + void (*free)(void*); + } callback_with_data; + struct { + const PageDirectory* page_directory; + u8* ptr; + size_t page_count; + } flush_tlb; + }; + + volatile bool async; + + ProcessorMessageEntry* per_proc_entries; +}; + +struct ProcessorMessageEntry { + ProcessorMessageEntry* next; + ProcessorMessage* msg; +}; + +struct DeferredCallEntry { + DeferredCallEntry* next; + union { + struct { + void (*handler)(); + } callback; + struct { + void* data; + void (*handler)(void*); + void (*free)(void*); + } callback_with_data; + }; + bool have_data; + bool was_allocated; +}; + +class Processor { + friend class ProcessorInfo; + + AK_MAKE_NONCOPYABLE(Processor); + AK_MAKE_NONMOVABLE(Processor); + + Processor* m_self; + + DescriptorTablePointer m_gdtr; + Descriptor m_gdt[256]; + u32 m_gdt_length; + + u32 m_cpu; + u32 m_in_irq; + Atomic<u32, AK::MemoryOrder::memory_order_relaxed> m_in_critical; + static Atomic<u32> s_idle_cpu_mask; + + TSS m_tss; + static FPUState s_clean_fpu_state; + CPUFeature m_features; + static volatile u32 g_total_processors; // atomic + u8 m_physical_address_bit_width; + + ProcessorInfo* m_info; + MemoryManagerData* m_mm_data; + SchedulerPerProcessorData* m_scheduler_data; + Thread* m_current_thread; + Thread* m_idle_thread; + + volatile ProcessorMessageEntry* m_message_queue; // atomic, LIFO + + bool m_invoke_scheduler_async; + bool m_scheduler_initialized; + Atomic<bool> m_halt_requested; + + DeferredCallEntry* m_pending_deferred_calls; // in reverse order + DeferredCallEntry* m_free_deferred_call_pool_entry; + DeferredCallEntry m_deferred_call_pool[5]; + + void gdt_init(); + void write_raw_gdt_entry(u16 selector, u32 low, u32 high); + void write_gdt_entry(u16 selector, Descriptor& descriptor); + static Vector<Processor*>& processors(); + + static void smp_return_to_pool(ProcessorMessage& msg); + static ProcessorMessage& smp_get_from_pool(); + static void smp_cleanup_message(ProcessorMessage& msg); + bool smp_queue_message(ProcessorMessage& msg); + static void smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async); + static void smp_broadcast_message(ProcessorMessage& msg); + static void smp_broadcast_wait_sync(ProcessorMessage& msg); + static void smp_broadcast_halt(); + + void deferred_call_pool_init(); + void deferred_call_execute_pending(); + DeferredCallEntry* deferred_call_get_free(); + void deferred_call_return_to_pool(DeferredCallEntry*); + void deferred_call_queue_entry(DeferredCallEntry*); + + void cpu_detect(); + void cpu_setup(); + + String features_string() const; + +public: + Processor() = default; + + void early_initialize(u32 cpu); + void initialize(u32 cpu); + + void idle_begin() + { + s_idle_cpu_mask.fetch_or(1u << m_cpu, AK::MemoryOrder::memory_order_relaxed); + } + + void idle_end() + { + s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed); + } + + static u32 count() + { + // NOTE: because this value never changes once all APs are booted, + // we don't really need to do an atomic_load() on this variable + return g_total_processors; + } + + ALWAYS_INLINE static void wait_check() + { + Processor::current().smp_process_pending_messages(); + // TODO: pause + } + + [[noreturn]] static void halt(); + + static void flush_entire_tlb_local() + { + write_cr3(read_cr3()); + } + + static void flush_tlb_local(VirtualAddress vaddr, size_t page_count); + static void flush_tlb(const PageDirectory*, VirtualAddress, size_t); + + Descriptor& get_gdt_entry(u16 selector); + void flush_gdt(); + const DescriptorTablePointer& get_gdtr(); + + static Processor& by_id(u32 cpu); + + static size_t processor_count() { return processors().size(); } + + template<typename Callback> + static inline IterationDecision for_each(Callback callback) + { + auto& procs = processors(); + size_t count = procs.size(); + for (size_t i = 0; i < count; i++) { + if (callback(*procs[i]) == IterationDecision::Break) + return IterationDecision::Break; + } + return IterationDecision::Continue; + } + + ALWAYS_INLINE u8 physical_address_bit_width() const { return m_physical_address_bit_width; } + + ALWAYS_INLINE ProcessorInfo& info() { return *m_info; } + + ALWAYS_INLINE static Processor& current() + { + return *(Processor*)read_fs_ptr(__builtin_offsetof(Processor, m_self)); + } + + ALWAYS_INLINE static bool is_initialized() + { + return get_fs() == GDT_SELECTOR_PROC && read_fs_u32(__builtin_offsetof(Processor, m_self)) != 0; + } + + ALWAYS_INLINE void set_scheduler_data(SchedulerPerProcessorData& scheduler_data) + { + m_scheduler_data = &scheduler_data; + } + + ALWAYS_INLINE SchedulerPerProcessorData& get_scheduler_data() const + { + return *m_scheduler_data; + } + + ALWAYS_INLINE void set_mm_data(MemoryManagerData& mm_data) + { + m_mm_data = &mm_data; + } + + ALWAYS_INLINE MemoryManagerData& get_mm_data() const + { + return *m_mm_data; + } + + ALWAYS_INLINE Thread* idle_thread() const + { + return m_idle_thread; + } + + ALWAYS_INLINE void set_idle_thread(Thread& idle_thread) + { + m_idle_thread = &idle_thread; + } + + ALWAYS_INLINE static Thread* current_thread() + { + // If we were to use Processor::current here, we'd have to + // disable interrupts to prevent a race where we may get pre-empted + // right after getting the Processor structure and then get moved + // to another processor, which would lead us to get the wrong thread. + // To avoid having to disable interrupts, we can just read the field + // directly in an atomic fashion, similar to Processor::current. + return (Thread*)read_fs_ptr(__builtin_offsetof(Processor, m_current_thread)); + } + + ALWAYS_INLINE static void set_current_thread(Thread& current_thread) + { + // See comment in Processor::current_thread + write_fs_u32(__builtin_offsetof(Processor, m_current_thread), FlatPtr(¤t_thread)); + } + + ALWAYS_INLINE u32 get_id() const + { + // NOTE: This variant should only be used when iterating over all + // Processor instances, or when it's guaranteed that the thread + // cannot move to another processor in between calling Processor::current + // and Processor::get_id, or if this fact is not important. + // All other cases should use Processor::id instead! + return m_cpu; + } + + ALWAYS_INLINE static u32 id() + { + // See comment in Processor::current_thread + return read_fs_ptr(__builtin_offsetof(Processor, m_cpu)); + } + + ALWAYS_INLINE u32 raise_irq() + { + return m_in_irq++; + } + + ALWAYS_INLINE void restore_irq(u32 prev_irq) + { + VERIFY(prev_irq <= m_in_irq); + if (!prev_irq) { + u32 prev_critical = 0; + if (m_in_critical.compare_exchange_strong(prev_critical, 1)) { + m_in_irq = prev_irq; + deferred_call_execute_pending(); + auto prev_raised = m_in_critical.exchange(prev_critical); + VERIFY(prev_raised == prev_critical + 1); + check_invoke_scheduler(); + } else if (prev_critical == 0) { + check_invoke_scheduler(); + } + } else { + m_in_irq = prev_irq; + } + } + + ALWAYS_INLINE u32& in_irq() + { + return m_in_irq; + } + + ALWAYS_INLINE void restore_in_critical(u32 critical) + { + m_in_critical = critical; + } + + ALWAYS_INLINE void enter_critical(u32& prev_flags) + { + prev_flags = cpu_flags(); + cli(); + m_in_critical++; + } + + ALWAYS_INLINE void leave_critical(u32 prev_flags) + { + cli(); // Need to prevent IRQs from interrupting us here! + VERIFY(m_in_critical > 0); + if (m_in_critical == 1) { + if (!m_in_irq) { + deferred_call_execute_pending(); + VERIFY(m_in_critical == 1); + } + m_in_critical--; + if (!m_in_irq) + check_invoke_scheduler(); + } else { + m_in_critical--; + } + if (prev_flags & 0x200) + sti(); + else + cli(); + } + + ALWAYS_INLINE u32 clear_critical(u32& prev_flags, bool enable_interrupts) + { + prev_flags = cpu_flags(); + u32 prev_crit = m_in_critical.exchange(0, AK::MemoryOrder::memory_order_acquire); + if (!m_in_irq) + check_invoke_scheduler(); + if (enable_interrupts) + sti(); + return prev_crit; + } + + ALWAYS_INLINE void restore_critical(u32 prev_crit, u32 prev_flags) + { + m_in_critical.store(prev_crit, AK::MemoryOrder::memory_order_release); + VERIFY(!prev_crit || !(prev_flags & 0x200)); + if (prev_flags & 0x200) + sti(); + else + cli(); + } + + ALWAYS_INLINE u32 in_critical() { return m_in_critical.load(); } + + ALWAYS_INLINE const FPUState& clean_fpu_state() const + { + return s_clean_fpu_state; + } + + static void smp_enable(); + bool smp_process_pending_messages(); + + template<typename Callback> + static void smp_broadcast(Callback callback, bool async) + { + auto* data = new Callback(move(callback)); + smp_broadcast( + [](void* data) { + (*reinterpret_cast<Callback*>(data))(); + }, + data, + [](void* data) { + delete reinterpret_cast<Callback*>(data); + }, + async); + } + static void smp_broadcast(void (*callback)(), bool async); + static void smp_broadcast(void (*callback)(void*), void* data, void (*free_data)(void*), bool async); + template<typename Callback> + static void smp_unicast(u32 cpu, Callback callback, bool async) + { + auto* data = new Callback(move(callback)); + smp_unicast( + cpu, + [](void* data) { + (*reinterpret_cast<Callback*>(data))(); + }, + data, + [](void* data) { + delete reinterpret_cast<Callback*>(data); + }, + async); + } + static void smp_unicast(u32 cpu, void (*callback)(), bool async); + static void smp_unicast(u32 cpu, void (*callback)(void*), void* data, void (*free_data)(void*), bool async); + static void smp_broadcast_flush_tlb(const PageDirectory*, VirtualAddress, size_t); + static u32 smp_wake_n_idle_processors(u32 wake_count); + + template<typename Callback> + static void deferred_call_queue(Callback callback) + { + auto* data = new Callback(move(callback)); + deferred_call_queue( + [](void* data) { + (*reinterpret_cast<Callback*>(data))(); + }, + data, + [](void* data) { + delete reinterpret_cast<Callback*>(data); + }); + } + static void deferred_call_queue(void (*callback)()); + static void deferred_call_queue(void (*callback)(void*), void* data, void (*free_data)(void*)); + + ALWAYS_INLINE bool has_feature(CPUFeature f) const + { + return (static_cast<u32>(m_features) & static_cast<u32>(f)) != 0; + } + + void check_invoke_scheduler(); + void invoke_scheduler_async() { m_invoke_scheduler_async = true; } + + void enter_trap(TrapFrame& trap, bool raise_irq); + + void exit_trap(TrapFrame& trap); + + [[noreturn]] void initialize_context_switching(Thread& initial_thread); + void switch_context(Thread*& from_thread, Thread*& to_thread); + [[noreturn]] static void assume_context(Thread& thread, FlatPtr flags); + u32 init_context(Thread& thread, bool leave_crit); + static Vector<FlatPtr> capture_stack_trace(Thread& thread, size_t max_frames = 0); + + String platform_string() const; +}; + +class ScopedCritical { + AK_MAKE_NONCOPYABLE(ScopedCritical); + +public: + ScopedCritical() + { + enter(); + } + + ~ScopedCritical() + { + if (m_valid) + leave(); + } + + ScopedCritical(ScopedCritical&& from) + : m_prev_flags(exchange(from.m_prev_flags, 0)) + , m_valid(exchange(from.m_valid, false)) + { + } + + ScopedCritical& operator=(ScopedCritical&& from) + { + if (&from != this) { + m_prev_flags = exchange(from.m_prev_flags, 0); + m_valid = exchange(from.m_valid, false); + } + return *this; + } + + void leave() + { + VERIFY(m_valid); + m_valid = false; + Processor::current().leave_critical(m_prev_flags); + } + + void enter() + { + VERIFY(!m_valid); + m_valid = true; + Processor::current().enter_critical(m_prev_flags); + } + +private: + u32 m_prev_flags { 0 }; + bool m_valid { false }; +}; + +struct TrapFrame { + u32 prev_irq_level; + TrapFrame* next_trap; + RegisterState* regs; // must be last + + TrapFrame() = delete; + TrapFrame(const TrapFrame&) = delete; + TrapFrame(TrapFrame&&) = delete; + TrapFrame& operator=(const TrapFrame&) = delete; + TrapFrame& operator=(TrapFrame&&) = delete; +}; + +#if ARCH(I386) +# define TRAP_FRAME_SIZE (3 * 4) +#else +# define TRAP_FRAME_SIZE (3 * 8) +#endif + +static_assert(TRAP_FRAME_SIZE == sizeof(TrapFrame)); + +extern "C" void enter_trap_no_irq(TrapFrame*); +extern "C" void enter_trap(TrapFrame*); +extern "C" void exit_trap(TrapFrame*); + +class MSR { + uint32_t m_msr; + +public: + static bool have() + { + CPUID id(1); + return (id.edx() & (1 << 5)) != 0; + } + + MSR(const MSR&) = delete; + MSR& operator=(const MSR&) = delete; + + MSR(uint32_t msr) + : m_msr(msr) + { + } + + void get(u32& low, u32& high) + { + asm volatile("rdmsr" + : "=a"(low), "=d"(high) + : "c"(m_msr)); + } + + void set(u32 low, u32 high) + { + asm volatile("wrmsr" ::"a"(low), "d"(high), "c"(m_msr)); + } +}; + +ALWAYS_INLINE void stac() +{ + if (!Processor::current().has_feature(CPUFeature::SMAP)) + return; + asm volatile("stac" :: + : "cc"); +} + +ALWAYS_INLINE void clac() +{ + if (!Processor::current().has_feature(CPUFeature::SMAP)) + return; + asm volatile("clac" :: + : "cc"); +} +} |