From 927926b9247522b4c8854d48a79b55599108cff2 Mon Sep 17 00:00:00 2001 From: Liav A Date: Fri, 24 Feb 2023 20:25:52 +0200 Subject: Kernel: Move Performance-measurement code to the Tasks subdirectory --- Kernel/Arch/PageFault.cpp | 2 +- Kernel/Arch/x86_64/Interrupts.cpp | 2 +- Kernel/CMakeLists.txt | 2 +- .../FileSystem/SysFS/Subsystems/Kernel/Profile.cpp | 2 +- Kernel/Heap/kmalloc.cpp | 2 +- Kernel/Memory/AddressSpace.cpp | 2 +- Kernel/PerformanceEventBuffer.cpp | 380 --------------------- Kernel/PerformanceEventBuffer.h | 155 --------- Kernel/PerformanceManager.h | 184 ---------- Kernel/Syscall.cpp | 2 +- Kernel/Syscalls/execve.cpp | 2 +- Kernel/Syscalls/exit.cpp | 2 +- Kernel/Syscalls/fork.cpp | 2 +- Kernel/Syscalls/mmap.cpp | 4 +- Kernel/Syscalls/perf_event.cpp | 2 +- Kernel/Syscalls/profiling.cpp | 2 +- Kernel/Syscalls/read.cpp | 2 +- Kernel/Syscalls/thread.cpp | 2 +- Kernel/Tasks/PerformanceEventBuffer.cpp | 380 +++++++++++++++++++++ Kernel/Tasks/PerformanceEventBuffer.h | 155 +++++++++ Kernel/Tasks/PerformanceManager.h | 184 ++++++++++ Kernel/Tasks/Process.cpp | 4 +- Kernel/Tasks/Process.h | 2 +- Kernel/Tasks/Scheduler.cpp | 2 +- Kernel/Tasks/Thread.cpp | 2 +- Kernel/Time/TimeManagement.cpp | 2 +- 26 files changed, 741 insertions(+), 741 deletions(-) delete mode 100644 Kernel/PerformanceEventBuffer.cpp delete mode 100644 Kernel/PerformanceEventBuffer.h delete mode 100644 Kernel/PerformanceManager.h create mode 100644 Kernel/Tasks/PerformanceEventBuffer.cpp create mode 100644 Kernel/Tasks/PerformanceEventBuffer.h create mode 100644 Kernel/Tasks/PerformanceManager.h diff --git a/Kernel/Arch/PageFault.cpp b/Kernel/Arch/PageFault.cpp index a341de455a..033bb41f3a 100644 --- a/Kernel/Arch/PageFault.cpp +++ b/Kernel/Arch/PageFault.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include namespace Kernel { diff --git a/Kernel/Arch/x86_64/Interrupts.cpp b/Kernel/Arch/x86_64/Interrupts.cpp index a210c22740..d43ffe940f 100644 --- a/Kernel/Arch/x86_64/Interrupts.cpp +++ b/Kernel/Arch/x86_64/Interrupts.cpp @@ -16,9 +16,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index 405a0d863a..c8d736ebc7 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -258,7 +258,6 @@ set(KERNEL_SOURCES Net/Socket.cpp Net/TCPSocket.cpp Net/UDPSocket.cpp - PerformanceEventBuffer.cpp Syscall.cpp Security/AddressSanitizer.cpp Security/Credentials.cpp @@ -347,6 +346,7 @@ set(KERNEL_SOURCES Tasks/CrashHandler.cpp Tasks/FinalizerTask.cpp Tasks/FutexQueue.cpp + Tasks/PerformanceEventBuffer.cpp Tasks/Process.cpp Tasks/ProcessGroup.cpp Tasks/ProcessList.cpp diff --git a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Profile.cpp b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Profile.cpp index 7a12ea0a03..51622d358d 100644 --- a/Kernel/FileSystem/SysFS/Subsystems/Kernel/Profile.cpp +++ b/Kernel/FileSystem/SysFS/Subsystems/Kernel/Profile.cpp @@ -5,8 +5,8 @@ */ #include -#include #include +#include namespace Kernel { diff --git a/Kernel/Heap/kmalloc.cpp b/Kernel/Heap/kmalloc.cpp index 5266a47b8f..9212c36d4c 100644 --- a/Kernel/Heap/kmalloc.cpp +++ b/Kernel/Heap/kmalloc.cpp @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #if ARCH(X86_64) || ARCH(AARCH64) static constexpr size_t CHUNK_SIZE = 64; diff --git a/Kernel/Memory/AddressSpace.cpp b/Kernel/Memory/AddressSpace.cpp index 643f6bd94a..0d972cd1cc 100644 --- a/Kernel/Memory/AddressSpace.cpp +++ b/Kernel/Memory/AddressSpace.cpp @@ -12,8 +12,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/Kernel/PerformanceEventBuffer.cpp b/Kernel/PerformanceEventBuffer.cpp deleted file mode 100644 index 929e569f0d..0000000000 --- a/Kernel/PerformanceEventBuffer.cpp +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright (c) 2020-2021, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Kernel { - -PerformanceEventBuffer::PerformanceEventBuffer(NonnullOwnPtr buffer) - : m_buffer(move(buffer)) -{ -} - -NEVER_INLINE ErrorOr PerformanceEventBuffer::append(int type, FlatPtr arg1, FlatPtr arg2, StringView arg3, Thread* current_thread, FlatPtr arg4, u64 arg5, ErrorOr const& arg6) -{ - FlatPtr base_pointer = (FlatPtr)__builtin_frame_address(0); - return append_with_ip_and_bp(current_thread->pid(), current_thread->tid(), 0, base_pointer, type, 0, arg1, arg2, arg3, arg4, arg5, arg6); -} - -static Vector raw_backtrace(FlatPtr bp, FlatPtr ip) -{ - Vector backtrace; - if (ip != 0) - backtrace.unchecked_append(ip); - FlatPtr stack_ptr_copy; - FlatPtr stack_ptr = bp; - // FIXME: Figure out how to remove this SmapDisabler without breaking profile stacks. - SmapDisabler disabler; - // NOTE: The stack should always have kernel frames first, followed by userspace frames. - // If a userspace frame points back into kernel memory, something is afoot. - bool is_walking_userspace_stack = false; - while (stack_ptr) { - void* fault_at; - if (!safe_memcpy(&stack_ptr_copy, (void*)stack_ptr, sizeof(FlatPtr), fault_at)) - break; - if (!Memory::is_user_address(VirtualAddress { stack_ptr })) { - if (is_walking_userspace_stack) { - dbgln("SHENANIGANS! Userspace stack points back into kernel memory"); - break; - } - } else { - is_walking_userspace_stack = true; - } - FlatPtr retaddr; - if (!safe_memcpy(&retaddr, (void*)(stack_ptr + sizeof(FlatPtr)), sizeof(FlatPtr), fault_at)) - break; - if (retaddr == 0) - break; - backtrace.unchecked_append(retaddr); - if (backtrace.size() == PerformanceEvent::max_stack_frame_count) - break; - stack_ptr = stack_ptr_copy; - } - return backtrace; -} - -ErrorOr PerformanceEventBuffer::append_with_ip_and_bp(ProcessID pid, ThreadID tid, RegisterState const& regs, - int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4, u64 arg5, ErrorOr const& arg6) -{ - return append_with_ip_and_bp(pid, tid, regs.ip(), regs.bp(), type, lost_samples, arg1, arg2, arg3, arg4, arg5, arg6); -} - -ErrorOr PerformanceEventBuffer::append_with_ip_and_bp(ProcessID pid, ThreadID tid, - FlatPtr ip, FlatPtr bp, int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4, u64 arg5, ErrorOr const& arg6) -{ - if (count() >= capacity()) - return ENOBUFS; - - if ((g_profiling_event_mask & type) == 0) - return EINVAL; - - auto* current_thread = Thread::current(); - u32 enter_count = 0; - if (current_thread) - enter_count = current_thread->enter_profiler(); - ScopeGuard leave_profiler([&] { - if (current_thread) - current_thread->leave_profiler(); - }); - if (enter_count > 0) - return EINVAL; - - PerformanceEvent event; - event.type = type; - event.lost_samples = lost_samples; - - switch (type) { - case PERF_EVENT_SAMPLE: - break; - case PERF_EVENT_MALLOC: - event.data.malloc.size = arg1; - event.data.malloc.ptr = arg2; - break; - case PERF_EVENT_FREE: - event.data.free.ptr = arg1; - break; - case PERF_EVENT_MMAP: - event.data.mmap.ptr = arg1; - event.data.mmap.size = arg2; - memset(event.data.mmap.name, 0, sizeof(event.data.mmap.name)); - if (!arg3.is_empty()) - memcpy(event.data.mmap.name, arg3.characters_without_null_termination(), min(arg3.length(), sizeof(event.data.mmap.name) - 1)); - break; - case PERF_EVENT_MUNMAP: - event.data.munmap.ptr = arg1; - event.data.munmap.size = arg2; - break; - case PERF_EVENT_PROCESS_CREATE: - event.data.process_create.parent_pid = arg1; - memset(event.data.process_create.executable, 0, sizeof(event.data.process_create.executable)); - if (!arg3.is_empty()) { - memcpy(event.data.process_create.executable, arg3.characters_without_null_termination(), - min(arg3.length(), sizeof(event.data.process_create.executable) - 1)); - } - break; - case PERF_EVENT_PROCESS_EXEC: - memset(event.data.process_exec.executable, 0, sizeof(event.data.process_exec.executable)); - if (!arg3.is_empty()) { - memcpy(event.data.process_exec.executable, arg3.characters_without_null_termination(), - min(arg3.length(), sizeof(event.data.process_exec.executable) - 1)); - } - break; - case PERF_EVENT_PROCESS_EXIT: - break; - case PERF_EVENT_THREAD_CREATE: - event.data.thread_create.parent_tid = arg1; - break; - case PERF_EVENT_THREAD_EXIT: - break; - case PERF_EVENT_CONTEXT_SWITCH: - event.data.context_switch.next_pid = arg1; - event.data.context_switch.next_tid = arg2; - break; - case PERF_EVENT_KMALLOC: - event.data.kmalloc.size = arg1; - event.data.kmalloc.ptr = arg2; - break; - case PERF_EVENT_KFREE: - event.data.kfree.size = arg1; - event.data.kfree.ptr = arg2; - break; - case PERF_EVENT_PAGE_FAULT: - break; - case PERF_EVENT_SYSCALL: - break; - case PERF_EVENT_SIGNPOST: - event.data.signpost.arg1 = arg1; - event.data.signpost.arg2 = arg2; - break; - case PERF_EVENT_READ: - event.data.read.fd = arg1; - event.data.read.size = arg2; - event.data.read.filename_index = arg4; - event.data.read.start_timestamp = arg5; - event.data.read.success = !arg6.is_error(); - break; - default: - return EINVAL; - } - - auto backtrace = raw_backtrace(bp, ip); - event.stack_size = min(sizeof(event.stack) / sizeof(FlatPtr), static_cast(backtrace.size())); - memcpy(event.stack, backtrace.data(), event.stack_size * sizeof(FlatPtr)); - - event.pid = pid.value(); - event.tid = tid.value(); - event.timestamp = TimeManagement::the().uptime_ms(); - at(m_count++) = event; - return {}; -} - -PerformanceEvent& PerformanceEventBuffer::at(size_t index) -{ - VERIFY(index < capacity()); - auto* events = reinterpret_cast(m_buffer->data()); - return events[index]; -} - -template -ErrorOr PerformanceEventBuffer::to_json_impl(Serializer& object) const -{ - { - auto strings = TRY(object.add_array("strings"sv)); - Vector strings_sorted_by_index; - TRY(strings_sorted_by_index.try_resize(m_strings.size())); - - for (auto& entry : m_strings) { - strings_sorted_by_index[entry.value] = const_cast(entry.key.ptr()); - } - - for (size_t i = 0; i < m_strings.size(); i++) { - TRY(strings.add(strings_sorted_by_index[i]->view())); - } - - TRY(strings.finish()); - } - - auto current_process_credentials = Process::current().credentials(); - bool show_kernel_addresses = current_process_credentials->is_superuser(); - auto array = TRY(object.add_array("events"sv)); - bool seen_first_sample = false; - for (size_t i = 0; i < m_count; ++i) { - auto const& event = at(i); - - if (!show_kernel_addresses) { - if (event.type == PERF_EVENT_KMALLOC || event.type == PERF_EVENT_KFREE) - continue; - } - - auto event_object = TRY(array.add_object()); - switch (event.type) { - case PERF_EVENT_SAMPLE: - TRY(event_object.add("type"sv, "sample")); - break; - case PERF_EVENT_MALLOC: - TRY(event_object.add("type"sv, "malloc")); - TRY(event_object.add("ptr"sv, static_cast(event.data.malloc.ptr))); - TRY(event_object.add("size"sv, static_cast(event.data.malloc.size))); - break; - case PERF_EVENT_FREE: - TRY(event_object.add("type"sv, "free")); - TRY(event_object.add("ptr"sv, static_cast(event.data.free.ptr))); - break; - case PERF_EVENT_MMAP: - TRY(event_object.add("type"sv, "mmap")); - TRY(event_object.add("ptr"sv, static_cast(event.data.mmap.ptr))); - TRY(event_object.add("size"sv, static_cast(event.data.mmap.size))); - TRY(event_object.add("name"sv, event.data.mmap.name)); - break; - case PERF_EVENT_MUNMAP: - TRY(event_object.add("type"sv, "munmap")); - TRY(event_object.add("ptr"sv, static_cast(event.data.munmap.ptr))); - TRY(event_object.add("size"sv, static_cast(event.data.munmap.size))); - break; - case PERF_EVENT_PROCESS_CREATE: - TRY(event_object.add("type"sv, "process_create")); - TRY(event_object.add("parent_pid"sv, static_cast(event.data.process_create.parent_pid))); - TRY(event_object.add("executable"sv, event.data.process_create.executable)); - break; - case PERF_EVENT_PROCESS_EXEC: - TRY(event_object.add("type"sv, "process_exec")); - TRY(event_object.add("executable"sv, event.data.process_exec.executable)); - break; - case PERF_EVENT_PROCESS_EXIT: - TRY(event_object.add("type"sv, "process_exit")); - break; - case PERF_EVENT_THREAD_CREATE: - TRY(event_object.add("type"sv, "thread_create")); - TRY(event_object.add("parent_tid"sv, static_cast(event.data.thread_create.parent_tid))); - break; - case PERF_EVENT_THREAD_EXIT: - TRY(event_object.add("type"sv, "thread_exit")); - break; - case PERF_EVENT_CONTEXT_SWITCH: - TRY(event_object.add("type"sv, "context_switch")); - TRY(event_object.add("next_pid"sv, static_cast(event.data.context_switch.next_pid))); - TRY(event_object.add("next_tid"sv, static_cast(event.data.context_switch.next_tid))); - break; - case PERF_EVENT_KMALLOC: - TRY(event_object.add("type"sv, "kmalloc")); - TRY(event_object.add("ptr"sv, static_cast(event.data.kmalloc.ptr))); - TRY(event_object.add("size"sv, static_cast(event.data.kmalloc.size))); - break; - case PERF_EVENT_KFREE: - TRY(event_object.add("type"sv, "kfree")); - TRY(event_object.add("ptr"sv, static_cast(event.data.kfree.ptr))); - TRY(event_object.add("size"sv, static_cast(event.data.kfree.size))); - break; - case PERF_EVENT_PAGE_FAULT: - TRY(event_object.add("type"sv, "page_fault")); - break; - case PERF_EVENT_SYSCALL: - TRY(event_object.add("type"sv, "syscall")); - break; - case PERF_EVENT_SIGNPOST: - TRY(event_object.add("type"sv, "signpost"sv)); - TRY(event_object.add("arg1"sv, event.data.signpost.arg1)); - TRY(event_object.add("arg2"sv, event.data.signpost.arg2)); - break; - case PERF_EVENT_READ: - TRY(event_object.add("type"sv, "read")); - TRY(event_object.add("fd"sv, event.data.read.fd)); - TRY(event_object.add("size"sv, event.data.read.size)); - TRY(event_object.add("filename_index"sv, event.data.read.filename_index)); - TRY(event_object.add("start_timestamp"sv, event.data.read.start_timestamp)); - TRY(event_object.add("success"sv, event.data.read.success)); - break; - } - TRY(event_object.add("pid"sv, event.pid)); - TRY(event_object.add("tid"sv, event.tid)); - TRY(event_object.add("timestamp"sv, event.timestamp)); - TRY(event_object.add("lost_samples"sv, seen_first_sample ? event.lost_samples : 0)); - if (event.type == PERF_EVENT_SAMPLE) - seen_first_sample = true; - auto stack_array = TRY(event_object.add_array("stack"sv)); - for (size_t j = 0; j < event.stack_size; ++j) { - auto address = event.stack[j]; - if (!show_kernel_addresses && !Memory::is_user_address(VirtualAddress { address })) - address = 0xdeadc0de; - TRY(stack_array.add(address)); - } - TRY(stack_array.finish()); - TRY(event_object.finish()); - } - TRY(array.finish()); - TRY(object.finish()); - return {}; -} - -ErrorOr PerformanceEventBuffer::to_json(KBufferBuilder& builder) const -{ - auto object = TRY(JsonObjectSerializer<>::try_create(builder)); - return to_json_impl(object); -} - -OwnPtr PerformanceEventBuffer::try_create_with_size(size_t buffer_size) -{ - auto buffer_or_error = KBuffer::try_create_with_size("Performance events"sv, buffer_size, Memory::Region::Access::ReadWrite, AllocationStrategy::AllocateNow); - if (buffer_or_error.is_error()) - return {}; - return adopt_own_if_nonnull(new (nothrow) PerformanceEventBuffer(buffer_or_error.release_value())); -} - -ErrorOr PerformanceEventBuffer::add_process(Process const& process, ProcessEventType event_type) -{ - OwnPtr executable; - if (process.executable()) { - executable = TRY(process.executable()->try_serialize_absolute_path()); - } else { - executable = TRY(process.name().with([&](auto& process_name) { - return KString::formatted("<{}>", process_name->view()); - })); - } - - TRY(append_with_ip_and_bp(process.pid(), 0, 0, 0, - event_type == ProcessEventType::Create ? PERF_EVENT_PROCESS_CREATE : PERF_EVENT_PROCESS_EXEC, - 0, process.pid().value(), 0, executable->view())); - - ErrorOr result; - process.for_each_thread([&](auto& thread) { - result = append_with_ip_and_bp(process.pid(), thread.tid().value(), - 0, 0, PERF_EVENT_THREAD_CREATE, 0, 0, 0, {}); - return result.is_error() ? IterationDecision::Break : IterationDecision::Continue; - }); - TRY(result); - - return process.address_space().with([&](auto& space) -> ErrorOr { - for (auto const& region : space->region_tree().regions()) { - TRY(append_with_ip_and_bp(process.pid(), 0, - 0, 0, PERF_EVENT_MMAP, 0, region.range().base().get(), region.range().size(), region.name())); - } - return {}; - }); -} - -ErrorOr PerformanceEventBuffer::register_string(NonnullOwnPtr string) -{ - auto it = m_strings.find(string); - if (it != m_strings.end()) { - return it->value; - } - - auto new_index = m_strings.size(); - TRY(m_strings.try_set(move(string), move(new_index))); - return new_index; -} - -} diff --git a/Kernel/PerformanceEventBuffer.h b/Kernel/PerformanceEventBuffer.h deleted file mode 100644 index 74233424be..0000000000 --- a/Kernel/PerformanceEventBuffer.h +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include - -namespace Kernel { - -class KBufferBuilder; -struct RegisterState; - -struct [[gnu::packed]] MallocPerformanceEvent { - size_t size; - FlatPtr ptr; -}; - -struct [[gnu::packed]] FreePerformanceEvent { - size_t size; - FlatPtr ptr; -}; - -struct [[gnu::packed]] MmapPerformanceEvent { - size_t size; - FlatPtr ptr; - char name[64]; -}; - -struct [[gnu::packed]] MunmapPerformanceEvent { - size_t size; - FlatPtr ptr; -}; - -struct [[gnu::packed]] ProcessCreatePerformanceEvent { - pid_t parent_pid; - char executable[64]; -}; - -struct [[gnu::packed]] ProcessExecPerformanceEvent { - char executable[64]; -}; - -struct [[gnu::packed]] ThreadCreatePerformanceEvent { - pid_t parent_tid; -}; - -struct [[gnu::packed]] ContextSwitchPerformanceEvent { - pid_t next_pid; - u32 next_tid; -}; - -struct [[gnu::packed]] KMallocPerformanceEvent { - size_t size; - FlatPtr ptr; -}; - -struct [[gnu::packed]] KFreePerformanceEvent { - size_t size; - FlatPtr ptr; -}; - -struct [[gnu::packed]] SignpostPerformanceEvent { - FlatPtr arg1; - FlatPtr arg2; -}; - -struct [[gnu::packed]] ReadPerformanceEvent { - int fd; - size_t size; - size_t filename_index; - size_t start_timestamp; - bool success; -}; - -struct [[gnu::packed]] PerformanceEvent { - u32 type { 0 }; - u8 stack_size { 0 }; - u32 pid { 0 }; - u32 tid { 0 }; - u64 timestamp; - u32 lost_samples; - union { - MallocPerformanceEvent malloc; - FreePerformanceEvent free; - MmapPerformanceEvent mmap; - MunmapPerformanceEvent munmap; - ProcessCreatePerformanceEvent process_create; - ProcessExecPerformanceEvent process_exec; - ThreadCreatePerformanceEvent thread_create; - ContextSwitchPerformanceEvent context_switch; - KMallocPerformanceEvent kmalloc; - KFreePerformanceEvent kfree; - SignpostPerformanceEvent signpost; - ReadPerformanceEvent read; - } data; - static constexpr size_t max_stack_frame_count = 64; - FlatPtr stack[max_stack_frame_count]; -}; - -enum class ProcessEventType { - Create, - Exec -}; - -class PerformanceEventBuffer { -public: - static OwnPtr try_create_with_size(size_t buffer_size); - - ErrorOr append(int type, FlatPtr arg1, FlatPtr arg2, StringView arg3, Thread* current_thread = Thread::current(), FlatPtr arg4 = 0, u64 arg5 = 0, ErrorOr const& arg6 = 0); - ErrorOr append_with_ip_and_bp(ProcessID pid, ThreadID tid, FlatPtr eip, FlatPtr ebp, - int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4 = 0, u64 arg5 = {}, ErrorOr const& arg6 = 0); - ErrorOr append_with_ip_and_bp(ProcessID pid, ThreadID tid, RegisterState const& regs, - int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4 = 0, u64 arg5 = {}, ErrorOr const& arg6 = 0); - - void clear() - { - m_count = 0; - } - - size_t capacity() const { return m_buffer->size() / sizeof(PerformanceEvent); } - size_t count() const { return m_count; } - PerformanceEvent const& at(size_t index) const - { - return const_cast(*this).at(index); - } - - ErrorOr to_json(KBufferBuilder&) const; - - ErrorOr add_process(Process const&, ProcessEventType event_type); - - ErrorOr register_string(NonnullOwnPtr); - -private: - explicit PerformanceEventBuffer(NonnullOwnPtr); - - template - ErrorOr to_json_impl(Serializer&) const; - - PerformanceEvent& at(size_t index); - - size_t m_count { 0 }; - NonnullOwnPtr m_buffer; - - HashMap, size_t> m_strings; -}; - -extern bool g_profiling_all_threads; -extern PerformanceEventBuffer* g_global_perf_events; -extern u64 g_profiling_event_mask; - -} diff --git a/Kernel/PerformanceManager.h b/Kernel/PerformanceManager.h deleted file mode 100644 index bf2c1f500b..0000000000 --- a/Kernel/PerformanceManager.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) 2021, Brian Gianforcaro - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include - -namespace Kernel { - -class PerformanceManager { -public: - static void add_process_created_event(Process& process) - { - if (g_profiling_all_threads) { - VERIFY(g_global_perf_events); - (void)g_global_perf_events->add_process(process, ProcessEventType::Create); - } - } - - static void add_process_exec_event(Process& process) - { - if (auto* event_buffer = process.current_perf_events_buffer()) { - (void)event_buffer->add_process(process, ProcessEventType::Exec); - } - } - - static void add_process_exit_event(Process& process) - { - if (g_profiling_all_threads) { - VERIFY(g_global_perf_events); - [[maybe_unused]] auto rc = g_global_perf_events->append_with_ip_and_bp( - process.pid(), 0, 0, 0, PERF_EVENT_PROCESS_EXIT, 0, 0, 0, {}); - } - } - - static void add_thread_created_event(Thread& thread) - { - if (thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto rc = event_buffer->append(PERF_EVENT_THREAD_CREATE, thread.tid().value(), 0, {}, &thread); - } - } - - static void add_thread_exit_event(Thread& thread) - { - // As an exception this doesn't check whether profiling is suppressed for - // the thread so we can record the thread_exit event anyway. - if (auto* event_buffer = thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto rc = event_buffer->append(PERF_EVENT_THREAD_EXIT, thread.tid().value(), 0, {}, &thread); - } - } - - static void add_cpu_sample_event(Thread& current_thread, RegisterState const& regs, u32 lost_time) - { - if (current_thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto rc = event_buffer->append_with_ip_and_bp( - current_thread.pid(), current_thread.tid(), regs, PERF_EVENT_SAMPLE, lost_time, 0, 0, {}); - } - } - - static void add_mmap_perf_event(Process& current_process, Memory::Region const& region) - { - if (auto* event_buffer = current_process.current_perf_events_buffer()) { - [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_MMAP, region.vaddr().get(), region.size(), region.name()); - } - } - - static void add_unmap_perf_event(Process& current_process, Memory::VirtualRange const& region) - { - if (auto* event_buffer = current_process.current_perf_events_buffer()) { - [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_MUNMAP, region.base().get(), region.size(), {}); - } - } - - static void add_context_switch_perf_event(Thread& current_thread, Thread& next_thread) - { - if (current_thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_CONTEXT_SWITCH, next_thread.pid().value(), next_thread.tid().value(), {}); - } - } - - static void add_kmalloc_perf_event(Thread& current_thread, size_t size, FlatPtr ptr) - { - if (current_thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_KMALLOC, size, ptr, {}); - } - } - - static void add_kfree_perf_event(Thread& current_thread, size_t size, FlatPtr ptr) - { - if (current_thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_KFREE, size, ptr, {}); - } - } - - static void add_page_fault_event(Thread& thread, RegisterState const& regs) - { - if (thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto rc = event_buffer->append_with_ip_and_bp( - thread.pid(), thread.tid(), regs, PERF_EVENT_PAGE_FAULT, 0, 0, 0, {}); - } - } - - static void add_syscall_event(Thread& thread, RegisterState const& regs) - { - if (thread.is_profiling_suppressed()) - return; - if (auto* event_buffer = thread.process().current_perf_events_buffer()) { - [[maybe_unused]] auto rc = event_buffer->append_with_ip_and_bp( - thread.pid(), thread.tid(), regs, PERF_EVENT_SYSCALL, 0, 0, 0, {}); - } - } - - static void add_read_event(Thread& thread, int fd, size_t size, OpenFileDescription const& file_description, u64 start_timestamp, ErrorOr const& result) - { - if (thread.is_profiling_suppressed()) - return; - - auto* event_buffer = thread.process().current_perf_events_buffer(); - if (event_buffer == nullptr) - return; - - size_t filepath_string_index; - - if (auto path = file_description.original_absolute_path(); !path.is_error()) { - auto registered_result = event_buffer->register_string(move(path.value())); - if (registered_result.is_error()) - return; - filepath_string_index = registered_result.value(); - } else if (auto pseudo_path = file_description.pseudo_path(); !pseudo_path.is_error()) { - auto registered_result = event_buffer->register_string(move(pseudo_path.value())); - if (registered_result.is_error()) - return; - filepath_string_index = registered_result.value(); - } else { - auto invalid_path_string = KString::try_create(""sv); // TODO: Performance, unnecessary allocations. - if (invalid_path_string.is_error()) - return; - auto registered_result = event_buffer->register_string(move(invalid_path_string.value())); - if (registered_result.is_error()) - return; - filepath_string_index = registered_result.value(); - } - - [[maybe_unused]] auto rc = event_buffer->append(PERF_EVENT_READ, fd, size, {}, &thread, filepath_string_index, start_timestamp, result); // wrong arguments - } - - static void timer_tick(RegisterState const& regs) - { - static UnixDateTime last_wakeup; - auto now = kgettimeofday(); - constexpr auto ideal_interval = Duration::from_microseconds(1000'000 / OPTIMAL_PROFILE_TICKS_PER_SECOND_RATE); - auto expected_wakeup = last_wakeup + ideal_interval; - auto delay = (now > expected_wakeup) ? now - expected_wakeup : Duration::from_microseconds(0); - last_wakeup = now; - auto* current_thread = Thread::current(); - // FIXME: We currently don't collect samples while idle. - // That will be an interesting mode to add in the future. :^) - if (!current_thread || current_thread == Processor::idle_thread()) - return; - - auto lost_samples = delay.to_microseconds() / ideal_interval.to_microseconds(); - PerformanceManager::add_cpu_sample_event(*current_thread, regs, lost_samples); - } -}; - -} diff --git a/Kernel/Syscall.cpp b/Kernel/Syscall.cpp index a3282e9bb7..c6130882cb 100644 --- a/Kernel/Syscall.cpp +++ b/Kernel/Syscall.cpp @@ -10,8 +10,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/Kernel/Syscalls/execve.cpp b/Kernel/Syscalls/execve.cpp index ebce643278..c18d11f0cd 100644 --- a/Kernel/Syscalls/execve.cpp +++ b/Kernel/Syscalls/execve.cpp @@ -16,8 +16,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/Kernel/Syscalls/exit.cpp b/Kernel/Syscalls/exit.cpp index e257f0ebb9..9911d5bd5b 100644 --- a/Kernel/Syscalls/exit.cpp +++ b/Kernel/Syscalls/exit.cpp @@ -5,7 +5,7 @@ */ #include -#include +#include #include #include diff --git a/Kernel/Syscalls/fork.cpp b/Kernel/Syscalls/fork.cpp index da478ed134..556d5e86e9 100644 --- a/Kernel/Syscalls/fork.cpp +++ b/Kernel/Syscalls/fork.cpp @@ -8,8 +8,8 @@ #include #include #include -#include #include +#include #include #include diff --git a/Kernel/Syscalls/mmap.cpp b/Kernel/Syscalls/mmap.cpp index 9ae33bd4a6..d296da6114 100644 --- a/Kernel/Syscalls/mmap.cpp +++ b/Kernel/Syscalls/mmap.cpp @@ -17,8 +17,8 @@ #include #include #include -#include -#include +#include +#include #include #include diff --git a/Kernel/Syscalls/perf_event.cpp b/Kernel/Syscalls/perf_event.cpp index 543d3d0224..a359c8e885 100644 --- a/Kernel/Syscalls/perf_event.cpp +++ b/Kernel/Syscalls/perf_event.cpp @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include +#include #include namespace Kernel { diff --git a/Kernel/Syscalls/profiling.cpp b/Kernel/Syscalls/profiling.cpp index c21d4c4afa..f8a3a13f1d 100644 --- a/Kernel/Syscalls/profiling.cpp +++ b/Kernel/Syscalls/profiling.cpp @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include #include +#include #include #include #include diff --git a/Kernel/Syscalls/read.cpp b/Kernel/Syscalls/read.cpp index b350e17691..6c18561db0 100644 --- a/Kernel/Syscalls/read.cpp +++ b/Kernel/Syscalls/read.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include namespace Kernel { diff --git a/Kernel/Syscalls/thread.cpp b/Kernel/Syscalls/thread.cpp index c2fbbd2485..7017012f85 100644 --- a/Kernel/Syscalls/thread.cpp +++ b/Kernel/Syscalls/thread.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include diff --git a/Kernel/Tasks/PerformanceEventBuffer.cpp b/Kernel/Tasks/PerformanceEventBuffer.cpp new file mode 100644 index 0000000000..7a67708e10 --- /dev/null +++ b/Kernel/Tasks/PerformanceEventBuffer.cpp @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2020-2021, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Kernel { + +PerformanceEventBuffer::PerformanceEventBuffer(NonnullOwnPtr buffer) + : m_buffer(move(buffer)) +{ +} + +NEVER_INLINE ErrorOr PerformanceEventBuffer::append(int type, FlatPtr arg1, FlatPtr arg2, StringView arg3, Thread* current_thread, FlatPtr arg4, u64 arg5, ErrorOr const& arg6) +{ + FlatPtr base_pointer = (FlatPtr)__builtin_frame_address(0); + return append_with_ip_and_bp(current_thread->pid(), current_thread->tid(), 0, base_pointer, type, 0, arg1, arg2, arg3, arg4, arg5, arg6); +} + +static Vector raw_backtrace(FlatPtr bp, FlatPtr ip) +{ + Vector backtrace; + if (ip != 0) + backtrace.unchecked_append(ip); + FlatPtr stack_ptr_copy; + FlatPtr stack_ptr = bp; + // FIXME: Figure out how to remove this SmapDisabler without breaking profile stacks. + SmapDisabler disabler; + // NOTE: The stack should always have kernel frames first, followed by userspace frames. + // If a userspace frame points back into kernel memory, something is afoot. + bool is_walking_userspace_stack = false; + while (stack_ptr) { + void* fault_at; + if (!safe_memcpy(&stack_ptr_copy, (void*)stack_ptr, sizeof(FlatPtr), fault_at)) + break; + if (!Memory::is_user_address(VirtualAddress { stack_ptr })) { + if (is_walking_userspace_stack) { + dbgln("SHENANIGANS! Userspace stack points back into kernel memory"); + break; + } + } else { + is_walking_userspace_stack = true; + } + FlatPtr retaddr; + if (!safe_memcpy(&retaddr, (void*)(stack_ptr + sizeof(FlatPtr)), sizeof(FlatPtr), fault_at)) + break; + if (retaddr == 0) + break; + backtrace.unchecked_append(retaddr); + if (backtrace.size() == PerformanceEvent::max_stack_frame_count) + break; + stack_ptr = stack_ptr_copy; + } + return backtrace; +} + +ErrorOr PerformanceEventBuffer::append_with_ip_and_bp(ProcessID pid, ThreadID tid, RegisterState const& regs, + int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4, u64 arg5, ErrorOr const& arg6) +{ + return append_with_ip_and_bp(pid, tid, regs.ip(), regs.bp(), type, lost_samples, arg1, arg2, arg3, arg4, arg5, arg6); +} + +ErrorOr PerformanceEventBuffer::append_with_ip_and_bp(ProcessID pid, ThreadID tid, + FlatPtr ip, FlatPtr bp, int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4, u64 arg5, ErrorOr const& arg6) +{ + if (count() >= capacity()) + return ENOBUFS; + + if ((g_profiling_event_mask & type) == 0) + return EINVAL; + + auto* current_thread = Thread::current(); + u32 enter_count = 0; + if (current_thread) + enter_count = current_thread->enter_profiler(); + ScopeGuard leave_profiler([&] { + if (current_thread) + current_thread->leave_profiler(); + }); + if (enter_count > 0) + return EINVAL; + + PerformanceEvent event; + event.type = type; + event.lost_samples = lost_samples; + + switch (type) { + case PERF_EVENT_SAMPLE: + break; + case PERF_EVENT_MALLOC: + event.data.malloc.size = arg1; + event.data.malloc.ptr = arg2; + break; + case PERF_EVENT_FREE: + event.data.free.ptr = arg1; + break; + case PERF_EVENT_MMAP: + event.data.mmap.ptr = arg1; + event.data.mmap.size = arg2; + memset(event.data.mmap.name, 0, sizeof(event.data.mmap.name)); + if (!arg3.is_empty()) + memcpy(event.data.mmap.name, arg3.characters_without_null_termination(), min(arg3.length(), sizeof(event.data.mmap.name) - 1)); + break; + case PERF_EVENT_MUNMAP: + event.data.munmap.ptr = arg1; + event.data.munmap.size = arg2; + break; + case PERF_EVENT_PROCESS_CREATE: + event.data.process_create.parent_pid = arg1; + memset(event.data.process_create.executable, 0, sizeof(event.data.process_create.executable)); + if (!arg3.is_empty()) { + memcpy(event.data.process_create.executable, arg3.characters_without_null_termination(), + min(arg3.length(), sizeof(event.data.process_create.executable) - 1)); + } + break; + case PERF_EVENT_PROCESS_EXEC: + memset(event.data.process_exec.executable, 0, sizeof(event.data.process_exec.executable)); + if (!arg3.is_empty()) { + memcpy(event.data.process_exec.executable, arg3.characters_without_null_termination(), + min(arg3.length(), sizeof(event.data.process_exec.executable) - 1)); + } + break; + case PERF_EVENT_PROCESS_EXIT: + break; + case PERF_EVENT_THREAD_CREATE: + event.data.thread_create.parent_tid = arg1; + break; + case PERF_EVENT_THREAD_EXIT: + break; + case PERF_EVENT_CONTEXT_SWITCH: + event.data.context_switch.next_pid = arg1; + event.data.context_switch.next_tid = arg2; + break; + case PERF_EVENT_KMALLOC: + event.data.kmalloc.size = arg1; + event.data.kmalloc.ptr = arg2; + break; + case PERF_EVENT_KFREE: + event.data.kfree.size = arg1; + event.data.kfree.ptr = arg2; + break; + case PERF_EVENT_PAGE_FAULT: + break; + case PERF_EVENT_SYSCALL: + break; + case PERF_EVENT_SIGNPOST: + event.data.signpost.arg1 = arg1; + event.data.signpost.arg2 = arg2; + break; + case PERF_EVENT_READ: + event.data.read.fd = arg1; + event.data.read.size = arg2; + event.data.read.filename_index = arg4; + event.data.read.start_timestamp = arg5; + event.data.read.success = !arg6.is_error(); + break; + default: + return EINVAL; + } + + auto backtrace = raw_backtrace(bp, ip); + event.stack_size = min(sizeof(event.stack) / sizeof(FlatPtr), static_cast(backtrace.size())); + memcpy(event.stack, backtrace.data(), event.stack_size * sizeof(FlatPtr)); + + event.pid = pid.value(); + event.tid = tid.value(); + event.timestamp = TimeManagement::the().uptime_ms(); + at(m_count++) = event; + return {}; +} + +PerformanceEvent& PerformanceEventBuffer::at(size_t index) +{ + VERIFY(index < capacity()); + auto* events = reinterpret_cast(m_buffer->data()); + return events[index]; +} + +template +ErrorOr PerformanceEventBuffer::to_json_impl(Serializer& object) const +{ + { + auto strings = TRY(object.add_array("strings"sv)); + Vector strings_sorted_by_index; + TRY(strings_sorted_by_index.try_resize(m_strings.size())); + + for (auto& entry : m_strings) { + strings_sorted_by_index[entry.value] = const_cast(entry.key.ptr()); + } + + for (size_t i = 0; i < m_strings.size(); i++) { + TRY(strings.add(strings_sorted_by_index[i]->view())); + } + + TRY(strings.finish()); + } + + auto current_process_credentials = Process::current().credentials(); + bool show_kernel_addresses = current_process_credentials->is_superuser(); + auto array = TRY(object.add_array("events"sv)); + bool seen_first_sample = false; + for (size_t i = 0; i < m_count; ++i) { + auto const& event = at(i); + + if (!show_kernel_addresses) { + if (event.type == PERF_EVENT_KMALLOC || event.type == PERF_EVENT_KFREE) + continue; + } + + auto event_object = TRY(array.add_object()); + switch (event.type) { + case PERF_EVENT_SAMPLE: + TRY(event_object.add("type"sv, "sample")); + break; + case PERF_EVENT_MALLOC: + TRY(event_object.add("type"sv, "malloc")); + TRY(event_object.add("ptr"sv, static_cast(event.data.malloc.ptr))); + TRY(event_object.add("size"sv, static_cast(event.data.malloc.size))); + break; + case PERF_EVENT_FREE: + TRY(event_object.add("type"sv, "free")); + TRY(event_object.add("ptr"sv, static_cast(event.data.free.ptr))); + break; + case PERF_EVENT_MMAP: + TRY(event_object.add("type"sv, "mmap")); + TRY(event_object.add("ptr"sv, static_cast(event.data.mmap.ptr))); + TRY(event_object.add("size"sv, static_cast(event.data.mmap.size))); + TRY(event_object.add("name"sv, event.data.mmap.name)); + break; + case PERF_EVENT_MUNMAP: + TRY(event_object.add("type"sv, "munmap")); + TRY(event_object.add("ptr"sv, static_cast(event.data.munmap.ptr))); + TRY(event_object.add("size"sv, static_cast(event.data.munmap.size))); + break; + case PERF_EVENT_PROCESS_CREATE: + TRY(event_object.add("type"sv, "process_create")); + TRY(event_object.add("parent_pid"sv, static_cast(event.data.process_create.parent_pid))); + TRY(event_object.add("executable"sv, event.data.process_create.executable)); + break; + case PERF_EVENT_PROCESS_EXEC: + TRY(event_object.add("type"sv, "process_exec")); + TRY(event_object.add("executable"sv, event.data.process_exec.executable)); + break; + case PERF_EVENT_PROCESS_EXIT: + TRY(event_object.add("type"sv, "process_exit")); + break; + case PERF_EVENT_THREAD_CREATE: + TRY(event_object.add("type"sv, "thread_create")); + TRY(event_object.add("parent_tid"sv, static_cast(event.data.thread_create.parent_tid))); + break; + case PERF_EVENT_THREAD_EXIT: + TRY(event_object.add("type"sv, "thread_exit")); + break; + case PERF_EVENT_CONTEXT_SWITCH: + TRY(event_object.add("type"sv, "context_switch")); + TRY(event_object.add("next_pid"sv, static_cast(event.data.context_switch.next_pid))); + TRY(event_object.add("next_tid"sv, static_cast(event.data.context_switch.next_tid))); + break; + case PERF_EVENT_KMALLOC: + TRY(event_object.add("type"sv, "kmalloc")); + TRY(event_object.add("ptr"sv, static_cast(event.data.kmalloc.ptr))); + TRY(event_object.add("size"sv, static_cast(event.data.kmalloc.size))); + break; + case PERF_EVENT_KFREE: + TRY(event_object.add("type"sv, "kfree")); + TRY(event_object.add("ptr"sv, static_cast(event.data.kfree.ptr))); + TRY(event_object.add("size"sv, static_cast(event.data.kfree.size))); + break; + case PERF_EVENT_PAGE_FAULT: + TRY(event_object.add("type"sv, "page_fault")); + break; + case PERF_EVENT_SYSCALL: + TRY(event_object.add("type"sv, "syscall")); + break; + case PERF_EVENT_SIGNPOST: + TRY(event_object.add("type"sv, "signpost"sv)); + TRY(event_object.add("arg1"sv, event.data.signpost.arg1)); + TRY(event_object.add("arg2"sv, event.data.signpost.arg2)); + break; + case PERF_EVENT_READ: + TRY(event_object.add("type"sv, "read")); + TRY(event_object.add("fd"sv, event.data.read.fd)); + TRY(event_object.add("size"sv, event.data.read.size)); + TRY(event_object.add("filename_index"sv, event.data.read.filename_index)); + TRY(event_object.add("start_timestamp"sv, event.data.read.start_timestamp)); + TRY(event_object.add("success"sv, event.data.read.success)); + break; + } + TRY(event_object.add("pid"sv, event.pid)); + TRY(event_object.add("tid"sv, event.tid)); + TRY(event_object.add("timestamp"sv, event.timestamp)); + TRY(event_object.add("lost_samples"sv, seen_first_sample ? event.lost_samples : 0)); + if (event.type == PERF_EVENT_SAMPLE) + seen_first_sample = true; + auto stack_array = TRY(event_object.add_array("stack"sv)); + for (size_t j = 0; j < event.stack_size; ++j) { + auto address = event.stack[j]; + if (!show_kernel_addresses && !Memory::is_user_address(VirtualAddress { address })) + address = 0xdeadc0de; + TRY(stack_array.add(address)); + } + TRY(stack_array.finish()); + TRY(event_object.finish()); + } + TRY(array.finish()); + TRY(object.finish()); + return {}; +} + +ErrorOr PerformanceEventBuffer::to_json(KBufferBuilder& builder) const +{ + auto object = TRY(JsonObjectSerializer<>::try_create(builder)); + return to_json_impl(object); +} + +OwnPtr PerformanceEventBuffer::try_create_with_size(size_t buffer_size) +{ + auto buffer_or_error = KBuffer::try_create_with_size("Performance events"sv, buffer_size, Memory::Region::Access::ReadWrite, AllocationStrategy::AllocateNow); + if (buffer_or_error.is_error()) + return {}; + return adopt_own_if_nonnull(new (nothrow) PerformanceEventBuffer(buffer_or_error.release_value())); +} + +ErrorOr PerformanceEventBuffer::add_process(Process const& process, ProcessEventType event_type) +{ + OwnPtr executable; + if (process.executable()) { + executable = TRY(process.executable()->try_serialize_absolute_path()); + } else { + executable = TRY(process.name().with([&](auto& process_name) { + return KString::formatted("<{}>", process_name->view()); + })); + } + + TRY(append_with_ip_and_bp(process.pid(), 0, 0, 0, + event_type == ProcessEventType::Create ? PERF_EVENT_PROCESS_CREATE : PERF_EVENT_PROCESS_EXEC, + 0, process.pid().value(), 0, executable->view())); + + ErrorOr result; + process.for_each_thread([&](auto& thread) { + result = append_with_ip_and_bp(process.pid(), thread.tid().value(), + 0, 0, PERF_EVENT_THREAD_CREATE, 0, 0, 0, {}); + return result.is_error() ? IterationDecision::Break : IterationDecision::Continue; + }); + TRY(result); + + return process.address_space().with([&](auto& space) -> ErrorOr { + for (auto const& region : space->region_tree().regions()) { + TRY(append_with_ip_and_bp(process.pid(), 0, + 0, 0, PERF_EVENT_MMAP, 0, region.range().base().get(), region.range().size(), region.name())); + } + return {}; + }); +} + +ErrorOr PerformanceEventBuffer::register_string(NonnullOwnPtr string) +{ + auto it = m_strings.find(string); + if (it != m_strings.end()) { + return it->value; + } + + auto new_index = m_strings.size(); + TRY(m_strings.try_set(move(string), move(new_index))); + return new_index; +} + +} diff --git a/Kernel/Tasks/PerformanceEventBuffer.h b/Kernel/Tasks/PerformanceEventBuffer.h new file mode 100644 index 0000000000..74233424be --- /dev/null +++ b/Kernel/Tasks/PerformanceEventBuffer.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2020, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +namespace Kernel { + +class KBufferBuilder; +struct RegisterState; + +struct [[gnu::packed]] MallocPerformanceEvent { + size_t size; + FlatPtr ptr; +}; + +struct [[gnu::packed]] FreePerformanceEvent { + size_t size; + FlatPtr ptr; +}; + +struct [[gnu::packed]] MmapPerformanceEvent { + size_t size; + FlatPtr ptr; + char name[64]; +}; + +struct [[gnu::packed]] MunmapPerformanceEvent { + size_t size; + FlatPtr ptr; +}; + +struct [[gnu::packed]] ProcessCreatePerformanceEvent { + pid_t parent_pid; + char executable[64]; +}; + +struct [[gnu::packed]] ProcessExecPerformanceEvent { + char executable[64]; +}; + +struct [[gnu::packed]] ThreadCreatePerformanceEvent { + pid_t parent_tid; +}; + +struct [[gnu::packed]] ContextSwitchPerformanceEvent { + pid_t next_pid; + u32 next_tid; +}; + +struct [[gnu::packed]] KMallocPerformanceEvent { + size_t size; + FlatPtr ptr; +}; + +struct [[gnu::packed]] KFreePerformanceEvent { + size_t size; + FlatPtr ptr; +}; + +struct [[gnu::packed]] SignpostPerformanceEvent { + FlatPtr arg1; + FlatPtr arg2; +}; + +struct [[gnu::packed]] ReadPerformanceEvent { + int fd; + size_t size; + size_t filename_index; + size_t start_timestamp; + bool success; +}; + +struct [[gnu::packed]] PerformanceEvent { + u32 type { 0 }; + u8 stack_size { 0 }; + u32 pid { 0 }; + u32 tid { 0 }; + u64 timestamp; + u32 lost_samples; + union { + MallocPerformanceEvent malloc; + FreePerformanceEvent free; + MmapPerformanceEvent mmap; + MunmapPerformanceEvent munmap; + ProcessCreatePerformanceEvent process_create; + ProcessExecPerformanceEvent process_exec; + ThreadCreatePerformanceEvent thread_create; + ContextSwitchPerformanceEvent context_switch; + KMallocPerformanceEvent kmalloc; + KFreePerformanceEvent kfree; + SignpostPerformanceEvent signpost; + ReadPerformanceEvent read; + } data; + static constexpr size_t max_stack_frame_count = 64; + FlatPtr stack[max_stack_frame_count]; +}; + +enum class ProcessEventType { + Create, + Exec +}; + +class PerformanceEventBuffer { +public: + static OwnPtr try_create_with_size(size_t buffer_size); + + ErrorOr append(int type, FlatPtr arg1, FlatPtr arg2, StringView arg3, Thread* current_thread = Thread::current(), FlatPtr arg4 = 0, u64 arg5 = 0, ErrorOr const& arg6 = 0); + ErrorOr append_with_ip_and_bp(ProcessID pid, ThreadID tid, FlatPtr eip, FlatPtr ebp, + int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4 = 0, u64 arg5 = {}, ErrorOr const& arg6 = 0); + ErrorOr append_with_ip_and_bp(ProcessID pid, ThreadID tid, RegisterState const& regs, + int type, u32 lost_samples, FlatPtr arg1, FlatPtr arg2, StringView arg3, FlatPtr arg4 = 0, u64 arg5 = {}, ErrorOr const& arg6 = 0); + + void clear() + { + m_count = 0; + } + + size_t capacity() const { return m_buffer->size() / sizeof(PerformanceEvent); } + size_t count() const { return m_count; } + PerformanceEvent const& at(size_t index) const + { + return const_cast(*this).at(index); + } + + ErrorOr to_json(KBufferBuilder&) const; + + ErrorOr add_process(Process const&, ProcessEventType event_type); + + ErrorOr register_string(NonnullOwnPtr); + +private: + explicit PerformanceEventBuffer(NonnullOwnPtr); + + template + ErrorOr to_json_impl(Serializer&) const; + + PerformanceEvent& at(size_t index); + + size_t m_count { 0 }; + NonnullOwnPtr m_buffer; + + HashMap, size_t> m_strings; +}; + +extern bool g_profiling_all_threads; +extern PerformanceEventBuffer* g_global_perf_events; +extern u64 g_profiling_event_mask; + +} diff --git a/Kernel/Tasks/PerformanceManager.h b/Kernel/Tasks/PerformanceManager.h new file mode 100644 index 0000000000..14a73f5d17 --- /dev/null +++ b/Kernel/Tasks/PerformanceManager.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2021, Brian Gianforcaro + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace Kernel { + +class PerformanceManager { +public: + static void add_process_created_event(Process& process) + { + if (g_profiling_all_threads) { + VERIFY(g_global_perf_events); + (void)g_global_perf_events->add_process(process, ProcessEventType::Create); + } + } + + static void add_process_exec_event(Process& process) + { + if (auto* event_buffer = process.current_perf_events_buffer()) { + (void)event_buffer->add_process(process, ProcessEventType::Exec); + } + } + + static void add_process_exit_event(Process& process) + { + if (g_profiling_all_threads) { + VERIFY(g_global_perf_events); + [[maybe_unused]] auto rc = g_global_perf_events->append_with_ip_and_bp( + process.pid(), 0, 0, 0, PERF_EVENT_PROCESS_EXIT, 0, 0, 0, {}); + } + } + + static void add_thread_created_event(Thread& thread) + { + if (thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto rc = event_buffer->append(PERF_EVENT_THREAD_CREATE, thread.tid().value(), 0, {}, &thread); + } + } + + static void add_thread_exit_event(Thread& thread) + { + // As an exception this doesn't check whether profiling is suppressed for + // the thread so we can record the thread_exit event anyway. + if (auto* event_buffer = thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto rc = event_buffer->append(PERF_EVENT_THREAD_EXIT, thread.tid().value(), 0, {}, &thread); + } + } + + static void add_cpu_sample_event(Thread& current_thread, RegisterState const& regs, u32 lost_time) + { + if (current_thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto rc = event_buffer->append_with_ip_and_bp( + current_thread.pid(), current_thread.tid(), regs, PERF_EVENT_SAMPLE, lost_time, 0, 0, {}); + } + } + + static void add_mmap_perf_event(Process& current_process, Memory::Region const& region) + { + if (auto* event_buffer = current_process.current_perf_events_buffer()) { + [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_MMAP, region.vaddr().get(), region.size(), region.name()); + } + } + + static void add_unmap_perf_event(Process& current_process, Memory::VirtualRange const& region) + { + if (auto* event_buffer = current_process.current_perf_events_buffer()) { + [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_MUNMAP, region.base().get(), region.size(), {}); + } + } + + static void add_context_switch_perf_event(Thread& current_thread, Thread& next_thread) + { + if (current_thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_CONTEXT_SWITCH, next_thread.pid().value(), next_thread.tid().value(), {}); + } + } + + static void add_kmalloc_perf_event(Thread& current_thread, size_t size, FlatPtr ptr) + { + if (current_thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_KMALLOC, size, ptr, {}); + } + } + + static void add_kfree_perf_event(Thread& current_thread, size_t size, FlatPtr ptr) + { + if (current_thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = current_thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto res = event_buffer->append(PERF_EVENT_KFREE, size, ptr, {}); + } + } + + static void add_page_fault_event(Thread& thread, RegisterState const& regs) + { + if (thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto rc = event_buffer->append_with_ip_and_bp( + thread.pid(), thread.tid(), regs, PERF_EVENT_PAGE_FAULT, 0, 0, 0, {}); + } + } + + static void add_syscall_event(Thread& thread, RegisterState const& regs) + { + if (thread.is_profiling_suppressed()) + return; + if (auto* event_buffer = thread.process().current_perf_events_buffer()) { + [[maybe_unused]] auto rc = event_buffer->append_with_ip_and_bp( + thread.pid(), thread.tid(), regs, PERF_EVENT_SYSCALL, 0, 0, 0, {}); + } + } + + static void add_read_event(Thread& thread, int fd, size_t size, OpenFileDescription const& file_description, u64 start_timestamp, ErrorOr const& result) + { + if (thread.is_profiling_suppressed()) + return; + + auto* event_buffer = thread.process().current_perf_events_buffer(); + if (event_buffer == nullptr) + return; + + size_t filepath_string_index; + + if (auto path = file_description.original_absolute_path(); !path.is_error()) { + auto registered_result = event_buffer->register_string(move(path.value())); + if (registered_result.is_error()) + return; + filepath_string_index = registered_result.value(); + } else if (auto pseudo_path = file_description.pseudo_path(); !pseudo_path.is_error()) { + auto registered_result = event_buffer->register_string(move(pseudo_path.value())); + if (registered_result.is_error()) + return; + filepath_string_index = registered_result.value(); + } else { + auto invalid_path_string = KString::try_create(""sv); // TODO: Performance, unnecessary allocations. + if (invalid_path_string.is_error()) + return; + auto registered_result = event_buffer->register_string(move(invalid_path_string.value())); + if (registered_result.is_error()) + return; + filepath_string_index = registered_result.value(); + } + + [[maybe_unused]] auto rc = event_buffer->append(PERF_EVENT_READ, fd, size, {}, &thread, filepath_string_index, start_timestamp, result); // wrong arguments + } + + static void timer_tick(RegisterState const& regs) + { + static UnixDateTime last_wakeup; + auto now = kgettimeofday(); + constexpr auto ideal_interval = Duration::from_microseconds(1000'000 / OPTIMAL_PROFILE_TICKS_PER_SECOND_RATE); + auto expected_wakeup = last_wakeup + ideal_interval; + auto delay = (now > expected_wakeup) ? now - expected_wakeup : Duration::from_microseconds(0); + last_wakeup = now; + auto* current_thread = Thread::current(); + // FIXME: We currently don't collect samples while idle. + // That will be an interesting mode to add in the future. :^) + if (!current_thread || current_thread == Processor::idle_thread()) + return; + + auto lost_samples = delay.to_microseconds() / ideal_interval.to_microseconds(); + PerformanceManager::add_cpu_sample_event(*current_thread, regs, lost_samples); + } +}; + +} diff --git a/Kernel/Tasks/Process.cpp b/Kernel/Tasks/Process.cpp index 7b8f348221..fcc74c5b86 100644 --- a/Kernel/Tasks/Process.cpp +++ b/Kernel/Tasks/Process.cpp @@ -31,10 +31,10 @@ #include #include #include -#include -#include #include #include +#include +#include #include #include #include diff --git a/Kernel/Tasks/Process.h b/Kernel/Tasks/Process.h index b94ff7d381..19ebfbb813 100644 --- a/Kernel/Tasks/Process.h +++ b/Kernel/Tasks/Process.h @@ -28,11 +28,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/Kernel/Tasks/Scheduler.cpp b/Kernel/Tasks/Scheduler.cpp index 8e2632c7c2..c5cd5539c0 100644 --- a/Kernel/Tasks/Scheduler.cpp +++ b/Kernel/Tasks/Scheduler.cpp @@ -12,8 +12,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/Kernel/Tasks/Thread.cpp b/Kernel/Tasks/Thread.cpp index d0cbc22e3c..3dd20af5db 100644 --- a/Kernel/Tasks/Thread.cpp +++ b/Kernel/Tasks/Thread.cpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/Kernel/Time/TimeManagement.cpp b/Kernel/Time/TimeManagement.cpp index 23d66b971c..003bc3fca5 100644 --- a/Kernel/Time/TimeManagement.cpp +++ b/Kernel/Time/TimeManagement.cpp @@ -26,8 +26,8 @@ #include #include #include -#include #include +#include #include #include #include -- cgit v1.2.3