diff options
Diffstat (limited to 'Kernel/Library')
24 files changed, 2187 insertions, 3 deletions
diff --git a/Kernel/Library/Assertions.h b/Kernel/Library/Assertions.h new file mode 100644 index 0000000000..aa0094f5e2 --- /dev/null +++ b/Kernel/Library/Assertions.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Platform.h> + +#define __STRINGIFY_HELPER(x) #x +#define __STRINGIFY(x) __STRINGIFY_HELPER(x) + +[[noreturn]] void __assertion_failed(char const* msg, char const* file, unsigned line, char const* func); +#define VERIFY(expr) \ + do { \ + if (!static_cast<bool>(expr)) [[unlikely]] \ + __assertion_failed(#expr, __FILE__, __LINE__, __PRETTY_FUNCTION__); \ + } while (0) + +#define VERIFY_NOT_REACHED() __assertion_failed("not reached", __FILE__, __LINE__, __PRETTY_FUNCTION__) + +extern "C" { +[[noreturn]] void _abort(); +[[noreturn]] void abort(); +} + +#define TODO() __assertion_failed("TODO", __FILE__, __LINE__, __PRETTY_FUNCTION__) +#define TODO_AARCH64() __assertion_failed("TODO_AARCH64", __FILE__, __LINE__, __PRETTY_FUNCTION__) + +#define VERIFY_INTERRUPTS_DISABLED() VERIFY(!(Processor::are_interrupts_enabled())) +#define VERIFY_INTERRUPTS_ENABLED() VERIFY(Processor::are_interrupts_enabled()) diff --git a/Kernel/Library/DoubleBuffer.cpp b/Kernel/Library/DoubleBuffer.cpp new file mode 100644 index 0000000000..b0624acaff --- /dev/null +++ b/Kernel/Library/DoubleBuffer.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/StringView.h> +#include <Kernel/InterruptDisabler.h> +#include <Kernel/Library/DoubleBuffer.h> + +namespace Kernel { + +inline void DoubleBuffer::compute_lockfree_metadata() +{ + InterruptDisabler disabler; + m_empty = m_read_buffer_index >= m_read_buffer->size && m_write_buffer->size == 0; + m_space_for_writing = m_capacity - m_write_buffer->size; +} + +ErrorOr<NonnullOwnPtr<DoubleBuffer>> DoubleBuffer::try_create(StringView name, size_t capacity) +{ + auto storage = TRY(KBuffer::try_create_with_size(name, capacity * 2, Memory::Region::Access::ReadWrite)); + return adopt_nonnull_own_or_enomem(new (nothrow) DoubleBuffer(capacity, move(storage))); +} + +DoubleBuffer::DoubleBuffer(size_t capacity, NonnullOwnPtr<KBuffer> storage) + : m_write_buffer(&m_buffer1) + , m_read_buffer(&m_buffer2) + , m_storage(move(storage)) + , m_capacity(capacity) +{ + m_buffer1.data = m_storage->data(); + m_buffer1.size = 0; + m_buffer2.data = m_storage->data() + capacity; + m_buffer2.size = 0; + m_space_for_writing = capacity; +} + +void DoubleBuffer::flip() +{ + VERIFY(m_read_buffer_index == m_read_buffer->size); + swap(m_read_buffer, m_write_buffer); + m_write_buffer->size = 0; + m_read_buffer_index = 0; + compute_lockfree_metadata(); +} + +ErrorOr<size_t> DoubleBuffer::write(UserOrKernelBuffer const& data, size_t size) +{ + if (!size) + return 0; + MutexLocker locker(m_lock); + size_t bytes_to_write = min(size, m_space_for_writing); + u8* write_ptr = m_write_buffer->data + m_write_buffer->size; + TRY(data.read(write_ptr, bytes_to_write)); + m_write_buffer->size += bytes_to_write; + compute_lockfree_metadata(); + if (m_unblock_callback && !m_empty) + m_unblock_callback(); + return bytes_to_write; +} + +ErrorOr<size_t> DoubleBuffer::read_impl(UserOrKernelBuffer& data, size_t size, MutexLocker&, bool advance_buffer_index) +{ + if (size == 0) + return 0; + if (m_read_buffer_index >= m_read_buffer->size && m_write_buffer->size != 0) + flip(); + if (m_read_buffer_index >= m_read_buffer->size) + return 0; + size_t nread = min(m_read_buffer->size - m_read_buffer_index, size); + TRY(data.write(m_read_buffer->data + m_read_buffer_index, nread)); + if (advance_buffer_index) + m_read_buffer_index += nread; + compute_lockfree_metadata(); + if (m_unblock_callback && m_space_for_writing > 0) + m_unblock_callback(); + return nread; +} + +ErrorOr<size_t> DoubleBuffer::read(UserOrKernelBuffer& data, size_t size) +{ + MutexLocker locker(m_lock); + return read_impl(data, size, locker, true); +} + +ErrorOr<size_t> DoubleBuffer::peek(UserOrKernelBuffer& data, size_t size) +{ + MutexLocker locker(m_lock); + return read_impl(data, size, locker, false); +} + +} diff --git a/Kernel/Library/DoubleBuffer.h b/Kernel/Library/DoubleBuffer.h new file mode 100644 index 0000000000..56e71deeab --- /dev/null +++ b/Kernel/Library/DoubleBuffer.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> +#include <Kernel/Library/KBuffer.h> +#include <Kernel/Library/UserOrKernelBuffer.h> +#include <Kernel/Locking/Mutex.h> +#include <Kernel/Tasks/Thread.h> + +namespace Kernel { + +class DoubleBuffer { +public: + static ErrorOr<NonnullOwnPtr<DoubleBuffer>> try_create(StringView name, size_t capacity = 65536); + ErrorOr<size_t> write(UserOrKernelBuffer const&, size_t); + ErrorOr<size_t> write(u8 const* data, size_t size) + { + return write(UserOrKernelBuffer::for_kernel_buffer(const_cast<u8*>(data)), size); + } + ErrorOr<size_t> read(UserOrKernelBuffer&, size_t); + ErrorOr<size_t> read(u8* data, size_t size) + { + auto buffer = UserOrKernelBuffer::for_kernel_buffer(data); + return read(buffer, size); + } + ErrorOr<size_t> peek(UserOrKernelBuffer&, size_t); + ErrorOr<size_t> peek(u8* data, size_t size) + { + auto buffer = UserOrKernelBuffer::for_kernel_buffer(data); + return peek(buffer, size); + } + + bool is_empty() const { return m_empty; } + + size_t space_for_writing() const { return m_space_for_writing; } + size_t immediately_readable() const + { + return (m_read_buffer->size - m_read_buffer_index) + m_write_buffer->size; + } + + void set_unblock_callback(Function<void()> callback) + { + VERIFY(!m_unblock_callback); + m_unblock_callback = move(callback); + } + +private: + explicit DoubleBuffer(size_t capacity, NonnullOwnPtr<KBuffer> storage); + void flip(); + void compute_lockfree_metadata(); + + ErrorOr<size_t> read_impl(UserOrKernelBuffer&, size_t, MutexLocker&, bool advance_buffer_index); + + struct InnerBuffer { + u8* data { nullptr }; + size_t size { 0 }; + }; + + InnerBuffer* m_write_buffer { nullptr }; + InnerBuffer* m_read_buffer { nullptr }; + InnerBuffer m_buffer1; + InnerBuffer m_buffer2; + + NonnullOwnPtr<KBuffer> m_storage; + Function<void()> m_unblock_callback; + size_t m_capacity { 0 }; + size_t m_read_buffer_index { 0 }; + size_t m_space_for_writing { 0 }; + bool m_empty { true }; + mutable Mutex m_lock { "DoubleBuffer"sv }; +}; + +} diff --git a/Kernel/Library/IOWindow.cpp b/Kernel/Library/IOWindow.cpp new file mode 100644 index 0000000000..b89b94148a --- /dev/null +++ b/Kernel/Library/IOWindow.cpp @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2022, Liav A. <liavalb@hotmail.co.il> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Bus/PCI/API.h> +#include <Kernel/Bus/PCI/Definitions.h> +#include <Kernel/Library/IOWindow.h> + +namespace Kernel { + +#if ARCH(X86_64) +ErrorOr<NonnullOwnPtr<IOWindow>> IOWindow::create_for_io_space(IOAddress address, u64 space_length) +{ + VERIFY(!Checked<u64>::addition_would_overflow(address.get(), space_length)); + auto io_address_range = TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOAddressData(address.get(), space_length))); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(io_address_range)))); +} + +IOWindow::IOWindow(NonnullOwnPtr<IOAddressData> io_range) + : m_space_type(SpaceType::IO) + , m_io_range(move(io_range)) +{ +} +#endif + +ErrorOr<NonnullOwnPtr<IOWindow>> IOWindow::create_from_io_window_with_offset(u64 offset, u64 space_length) +{ +#if ARCH(X86_64) + if (m_space_type == SpaceType::IO) { + VERIFY(m_io_range); + if (Checked<u64>::addition_would_overflow(m_io_range->address(), space_length)) + return Error::from_errno(EOVERFLOW); + auto io_address_range = TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOAddressData(as_io_address().offset(offset).get(), space_length))); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(io_address_range)))); + } +#endif + VERIFY(space_type() == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + + if (Checked<u64>::addition_would_overflow(m_memory_mapped_range->paddr.get(), offset)) + return Error::from_errno(EOVERFLOW); + if (Checked<u64>::addition_would_overflow(m_memory_mapped_range->paddr.get() + offset, space_length)) + return Error::from_errno(EOVERFLOW); + + auto memory_mapped_range = TRY(Memory::adopt_new_nonnull_own_typed_mapping<u8 volatile>(m_memory_mapped_range->paddr.offset(offset), space_length, Memory::Region::Access::ReadWrite)); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(memory_mapped_range)))); +} + +ErrorOr<NonnullOwnPtr<IOWindow>> IOWindow::create_from_io_window_with_offset(u64 offset) +{ + +#if ARCH(X86_64) + if (m_space_type == SpaceType::IO) { + VERIFY(m_io_range); + VERIFY(m_io_range->space_length() >= offset); + return create_from_io_window_with_offset(offset, m_io_range->space_length() - offset); + } +#endif + VERIFY(space_type() == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + VERIFY(m_memory_mapped_range->length >= offset); + return create_from_io_window_with_offset(offset, m_memory_mapped_range->length - offset); +} + +ErrorOr<NonnullOwnPtr<IOWindow>> IOWindow::create_for_pci_device_bar(PCI::DeviceIdentifier const& pci_device_identifier, PCI::HeaderType0BaseRegister pci_bar, u64 space_length) +{ + u64 pci_bar_value = PCI::get_BAR(pci_device_identifier, pci_bar); + auto pci_bar_space_type = PCI::get_BAR_space_type(pci_bar_value); + if (pci_bar_space_type == PCI::BARSpaceType::Memory64BitSpace) { + // FIXME: In theory, BAR5 cannot be assigned to 64 bit as it is the last one... + // however, there might be 64 bit BAR5 for real bare metal hardware, so remove this + // if it makes a problem. + if (pci_bar == PCI::HeaderType0BaseRegister::BAR5) { + return Error::from_errno(EINVAL); + } + u64 next_pci_bar_value = PCI::get_BAR(pci_device_identifier, static_cast<PCI::HeaderType0BaseRegister>(to_underlying(pci_bar) + 1)); + pci_bar_value |= next_pci_bar_value << 32; + } + + auto pci_bar_space_size = PCI::get_BAR_space_size(pci_device_identifier, pci_bar); + if (pci_bar_space_size < space_length) + return Error::from_errno(EIO); + + if (pci_bar_space_type == PCI::BARSpaceType::IOSpace) { +#if ARCH(X86_64) + if (Checked<u64>::addition_would_overflow(pci_bar_value, space_length)) + return Error::from_errno(EOVERFLOW); + auto io_address_range = TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOAddressData((pci_bar_value & 0xfffffffc), space_length))); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(io_address_range)))); +#else + // Note: For non-x86 platforms, IO PCI BARs are simply not useable. + return Error::from_errno(ENOTSUP); +#endif + } + + if (pci_bar_space_type == PCI::BARSpaceType::Memory32BitSpace && Checked<u32>::addition_would_overflow(pci_bar_value, space_length)) + return Error::from_errno(EOVERFLOW); + if (pci_bar_space_type == PCI::BARSpaceType::Memory16BitSpace && Checked<u16>::addition_would_overflow(pci_bar_value, space_length)) + return Error::from_errno(EOVERFLOW); + if (pci_bar_space_type == PCI::BARSpaceType::Memory64BitSpace && Checked<u64>::addition_would_overflow(pci_bar_value, space_length)) + return Error::from_errno(EOVERFLOW); + auto memory_mapped_range = TRY(Memory::adopt_new_nonnull_own_typed_mapping<u8 volatile>(PhysicalAddress(pci_bar_value & PCI::bar_address_mask), space_length, Memory::Region::Access::ReadWrite)); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) IOWindow(move(memory_mapped_range)))); +} + +ErrorOr<NonnullOwnPtr<IOWindow>> IOWindow::create_for_pci_device_bar(PCI::DeviceIdentifier const& pci_device_identifier, PCI::HeaderType0BaseRegister pci_bar) +{ + u64 pci_bar_space_size = PCI::get_BAR_space_size(pci_device_identifier, pci_bar); + return create_for_pci_device_bar(pci_device_identifier, pci_bar, pci_bar_space_size); +} + +IOWindow::IOWindow(NonnullOwnPtr<Memory::TypedMapping<u8 volatile>> memory_mapped_range) + : m_space_type(SpaceType::Memory) + , m_memory_mapped_range(move(memory_mapped_range)) +{ +} + +IOWindow::~IOWindow() = default; + +bool IOWindow::is_access_aligned(u64 offset, size_t byte_size_access) const +{ + return (offset % byte_size_access) == 0; +} + +bool IOWindow::is_access_in_range(u64 offset, size_t byte_size_access) const +{ + if (Checked<u64>::addition_would_overflow(offset, byte_size_access)) + return false; +#if ARCH(X86_64) + if (m_space_type == SpaceType::IO) { + VERIFY(m_io_range); + VERIFY(!Checked<u64>::addition_would_overflow(m_io_range->address(), m_io_range->space_length())); + // To understand how we treat IO address space with the corresponding calculation, the Intel Software Developer manual + // helps us to understand the layout of the IO address space - + // + // Intel® 64 and IA-32 Architectures Software Developer’s Manual, Volume 1: Basic Architecture, 16.3 I/O ADDRESS SPACE, page 16-1 wrote: + // Any two consecutive 8-bit ports can be treated as a 16-bit port, and any four consecutive ports can be a 32-bit port. + // In this manner, the processor can transfer 8, 16, or 32 bits to or from a device in the I/O address space. + // Like words in memory, 16-bit ports should be aligned to even addresses (0, 2, 4, ...) so that all 16 bits can be transferred in a single bus cycle. + // Likewise, 32-bit ports should be aligned to addresses that are multiples of four (0, 4, 8, ...). + // The processor supports data transfers to unaligned ports, but there is a performance penalty because one or more + // extra bus cycle must be used. + return (m_io_range->address() + m_io_range->space_length()) >= (offset + byte_size_access); + } +#endif + VERIFY(space_type() == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + VERIFY(!Checked<u64>::addition_would_overflow(m_memory_mapped_range->offset, m_memory_mapped_range->length)); + return (m_memory_mapped_range->offset + m_memory_mapped_range->length) >= (offset + byte_size_access); +} + +u8 IOWindow::read8(u64 offset) +{ + VERIFY(is_access_in_range(offset, sizeof(u8))); + u8 data { 0 }; + in<u8>(offset, data); + return data; +} +u16 IOWindow::read16(u64 offset) +{ + // Note: Although it might be OK to allow unaligned access on regular memory, + // for memory mapped IO access, it should always be considered a bug. + // The same goes for port mapped IO access, because in x86 unaligned access to ports + // is possible but there's a performance penalty. + VERIFY(is_access_in_range(offset, sizeof(u16))); + VERIFY(is_access_aligned(offset, sizeof(u16))); + u16 data { 0 }; + in<u16>(offset, data); + return data; +} +u32 IOWindow::read32(u64 offset) +{ + // Note: Although it might be OK to allow unaligned access on regular memory, + // for memory mapped IO access, it should always be considered a bug. + // The same goes for port mapped IO access, because in x86 unaligned access to ports + // is possible but there's a performance penalty. + VERIFY(is_access_in_range(offset, sizeof(u32))); + VERIFY(is_access_aligned(offset, sizeof(u32))); + u32 data { 0 }; + in<u32>(offset, data); + return data; +} + +void IOWindow::write8(u64 offset, u8 data) +{ + VERIFY(is_access_in_range(offset, sizeof(u8))); + out<u8>(offset, data); +} +void IOWindow::write16(u64 offset, u16 data) +{ + // Note: Although it might be OK to allow unaligned access on regular memory, + // for memory mapped IO access, it should always be considered a bug. + // The same goes for port mapped IO access, because in x86 unaligned access to ports + // is possible but there's a performance penalty. + VERIFY(is_access_in_range(offset, sizeof(u16))); + VERIFY(is_access_aligned(offset, sizeof(u16))); + out<u16>(offset, data); +} +void IOWindow::write32(u64 offset, u32 data) +{ + // Note: Although it might be OK to allow unaligned access on regular memory, + // for memory mapped IO access, it should always be considered a bug. + // The same goes for port mapped IO access, because in x86 unaligned access to ports + // is possible but there's a performance penalty. + VERIFY(is_access_in_range(offset, sizeof(u32))); + VERIFY(is_access_aligned(offset, sizeof(u32))); + out<u32>(offset, data); +} + +void IOWindow::write32_unaligned(u64 offset, u32 data) +{ + // Note: We only verify that we access IO in the expected range. + // Note: for port mapped IO access, because in x86 unaligned access to ports + // is possible but there's a performance penalty, we can still allow that to happen. + // However, it should be noted that most cases should not use unaligned access + // to hardware IO, so this is a valid case in emulators or hypervisors only. + // Note: Using this for memory mapped IO will fail for unaligned access, because + // there's no valid use case for it (yet). + VERIFY(space_type() != SpaceType::Memory); + VERIFY(is_access_in_range(offset, sizeof(u32))); + out<u32>(offset, data); +} + +u32 IOWindow::read32_unaligned(u64 offset) +{ + // Note: We only verify that we access IO in the expected range. + // Note: for port mapped IO access, because in x86 unaligned access to ports + // is possible but there's a performance penalty, we can still allow that to happen. + // However, it should be noted that most cases should not use unaligned access + // to hardware IO, so this is a valid case in emulators or hypervisors only. + // Note: Using this for memory mapped IO will fail for unaligned access, because + // there's no valid use case for it (yet). + VERIFY(space_type() != SpaceType::Memory); + VERIFY(is_access_in_range(offset, sizeof(u32))); + u32 data { 0 }; + in<u32>(offset, data); + return data; +} + +PhysicalAddress IOWindow::as_physical_memory_address() const +{ + VERIFY(space_type() == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + return m_memory_mapped_range->paddr; +} + +u8 volatile* IOWindow::as_memory_address_pointer() +{ + VERIFY(space_type() == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + return m_memory_mapped_range->ptr(); +} + +#if ARCH(X86_64) +IOAddress IOWindow::as_io_address() const +{ + VERIFY(space_type() == SpaceType::IO); + VERIFY(m_io_range); + return IOAddress(m_io_range->address()); +} +#endif + +} diff --git a/Kernel/Library/IOWindow.h b/Kernel/Library/IOWindow.h new file mode 100644 index 0000000000..d07793681b --- /dev/null +++ b/Kernel/Library/IOWindow.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2022, Liav A. <liavalb@hotmail.co.il> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/ByteReader.h> +#include <AK/Platform.h> +#include <AK/Types.h> +#if ARCH(X86_64) +# include <Kernel/Arch/x86_64/IO.h> +#endif +#include <Kernel/Bus/PCI/Definitions.h> +#include <Kernel/Memory/PhysicalAddress.h> +#include <Kernel/Memory/TypedMapping.h> + +namespace Kernel { + +class IOWindow { +public: + enum class SpaceType { +#if ARCH(X86_64) + IO, +#endif + Memory, + }; + + SpaceType space_type() const { return m_space_type; } + +#if ARCH(X86_64) + static ErrorOr<NonnullOwnPtr<IOWindow>> create_for_io_space(IOAddress, u64 space_length); +#endif + static ErrorOr<NonnullOwnPtr<IOWindow>> create_for_pci_device_bar(PCI::DeviceIdentifier const&, PCI::HeaderType0BaseRegister, u64 space_length); + static ErrorOr<NonnullOwnPtr<IOWindow>> create_for_pci_device_bar(PCI::DeviceIdentifier const&, PCI::HeaderType0BaseRegister); + + ErrorOr<NonnullOwnPtr<IOWindow>> create_from_io_window_with_offset(u64 offset, u64 space_length); + ErrorOr<NonnullOwnPtr<IOWindow>> create_from_io_window_with_offset(u64 offset); + + u8 read8(u64 offset); + u16 read16(u64 offset); + u32 read32(u64 offset); + + void write8(u64 offset, u8); + void write16(u64 offset, u16); + void write32(u64 offset, u32); + + // Note: These methods are useful in exceptional cases where we need to do unaligned + // access. This mostly happens on emulators and hypervisors (such as VMWare) because they don't enforce aligned access + // to IO and sometimes even require such access, so we have to use these functions. + void write32_unaligned(u64 offset, u32); + u32 read32_unaligned(u64 offset); + + bool operator==(IOWindow const& other) const = delete; + bool operator!=(IOWindow const& other) const = delete; + bool operator>(IOWindow const& other) const = delete; + bool operator>=(IOWindow const& other) const = delete; + bool operator<(IOWindow const& other) const = delete; + bool operator<=(IOWindow const& other) const = delete; + + ~IOWindow(); + + PhysicalAddress as_physical_memory_address() const; +#if ARCH(X86_64) + IOAddress as_io_address() const; +#endif + +private: + explicit IOWindow(NonnullOwnPtr<Memory::TypedMapping<u8 volatile>>); + + u8 volatile* as_memory_address_pointer(); + +#if ARCH(X86_64) + struct IOAddressData { + public: + IOAddressData(u64 address, u64 space_length) + : m_address(address) + , m_space_length(space_length) + { + } + u64 address() const { return m_address; } + u64 space_length() const { return m_space_length; } + + private: + u64 m_address { 0 }; + u64 m_space_length { 0 }; + }; + + explicit IOWindow(NonnullOwnPtr<IOAddressData>); +#endif + + bool is_access_in_range(u64 offset, size_t byte_size_access) const; + bool is_access_aligned(u64 offset, size_t byte_size_access) const; + + template<typename T> + ALWAYS_INLINE void in(u64 start_offset, T& data) + { +#if ARCH(X86_64) + if (m_space_type == SpaceType::IO) { + data = as_io_address().offset(start_offset).in<T>(); + return; + } +#endif + VERIFY(m_space_type == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + // Note: For memory-mapped IO we simply never allow unaligned access as it + // can cause problems with strict bare metal hardware. For example, some XHCI USB controllers + // might completely lock up because of an unaligned memory access to their registers. + VERIFY((start_offset % sizeof(T)) == 0); + data = *(T volatile*)(as_memory_address_pointer() + start_offset); + } + + template<typename T> + ALWAYS_INLINE void out(u64 start_offset, T value) + { +#if ARCH(X86_64) + if (m_space_type == SpaceType::IO) { + VERIFY(m_io_range); + as_io_address().offset(start_offset).out<T>(value); + return; + } +#endif + VERIFY(m_space_type == SpaceType::Memory); + VERIFY(m_memory_mapped_range); + // Note: For memory-mapped IO we simply never allow unaligned access as it + // can cause problems with strict bare metal hardware. For example, some XHCI USB controllers + // might completely lock up because of an unaligned memory access to their registers. + VERIFY((start_offset % sizeof(T)) == 0); + *(T volatile*)(as_memory_address_pointer() + start_offset) = value; + } + + SpaceType m_space_type { SpaceType::Memory }; + + OwnPtr<Memory::TypedMapping<u8 volatile>> m_memory_mapped_range; + +#if ARCH(X86_64) + OwnPtr<IOAddressData> m_io_range; +#endif +}; + +} + +template<> +struct AK::Formatter<Kernel::IOWindow> : AK::Formatter<FormatString> { + ErrorOr<void> format(FormatBuilder& builder, Kernel::IOWindow const& value) + { +#if ARCH(X86_64) + if (value.space_type() == Kernel::IOWindow::SpaceType::IO) + return Formatter<FormatString>::format(builder, "{}"sv, value.as_io_address()); +#endif + VERIFY(value.space_type() == Kernel::IOWindow::SpaceType::Memory); + return Formatter<FormatString>::format(builder, "Memory {}"sv, value.as_physical_memory_address()); + } +}; diff --git a/Kernel/Library/KBuffer.h b/Kernel/Library/KBuffer.h new file mode 100644 index 0000000000..b5694b3a92 --- /dev/null +++ b/Kernel/Library/KBuffer.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +// KBuffer: Memory buffer backed by a kernel region. +// +// The memory is allocated via the global kernel-only page allocator, rather than via +// kmalloc() which is what ByteBuffer/Vector/etc will use. +// +// This makes KBuffer a little heavier to allocate, but much better for large and/or +// long-lived allocations, since they don't put all that weight and pressure on the +// severely limited kmalloc heap. + +#include <AK/Assertions.h> +#include <AK/StringView.h> +#include <Kernel/Library/StdLib.h> // For memcpy. FIXME: Make memcpy less expensive to access a declaration of in the Kernel. +#include <Kernel/Memory/MemoryManager.h> + +namespace Kernel { + +class [[nodiscard]] KBuffer { +public: + static ErrorOr<NonnullOwnPtr<KBuffer>> try_create_with_size(StringView name, size_t size, Memory::Region::Access access = Memory::Region::Access::ReadWrite, AllocationStrategy strategy = AllocationStrategy::Reserve) + { + auto rounded_size = TRY(Memory::page_round_up(size)); + auto region = TRY(MM.allocate_kernel_region(rounded_size, name, access, strategy)); + return TRY(adopt_nonnull_own_or_enomem(new (nothrow) KBuffer { size, move(region) })); + } + + static ErrorOr<NonnullOwnPtr<KBuffer>> try_create_with_bytes(StringView name, ReadonlyBytes bytes, Memory::Region::Access access = Memory::Region::Access::ReadWrite, AllocationStrategy strategy = AllocationStrategy::Reserve) + { + auto buffer = TRY(try_create_with_size(name, bytes.size(), access, strategy)); + memcpy(buffer->data(), bytes.data(), bytes.size()); + return buffer; + } + + [[nodiscard]] u8* data() { return m_region->vaddr().as_ptr(); } + [[nodiscard]] u8 const* data() const { return m_region->vaddr().as_ptr(); } + [[nodiscard]] size_t size() const { return m_size; } + [[nodiscard]] size_t capacity() const { return m_region->size(); } + + [[nodiscard]] ReadonlyBytes bytes() const { return { data(), size() }; } + [[nodiscard]] Bytes bytes() { return { data(), size() }; } + + void set_size(size_t size) + { + VERIFY(size <= capacity()); + m_size = size; + } + +private: + explicit KBuffer(size_t size, NonnullOwnPtr<Memory::Region> region) + : m_size(size) + , m_region(move(region)) + { + } + + size_t m_size { 0 }; + NonnullOwnPtr<Memory::Region> m_region; +}; + +} diff --git a/Kernel/Library/KBufferBuilder.cpp b/Kernel/Library/KBufferBuilder.cpp new file mode 100644 index 0000000000..e772096aff --- /dev/null +++ b/Kernel/Library/KBufferBuilder.cpp @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/StdLibExtras.h> +#include <Kernel/Library/KBufferBuilder.h> + +namespace Kernel { + +inline bool KBufferBuilder::check_expand(size_t size) +{ + if (!m_buffer) + return false; + if ((m_size + size) < m_buffer->capacity()) + return true; + if (Checked<size_t>::addition_would_overflow(m_size, size)) + return false; + size_t new_buffer_size = m_size + size; + if (Checked<size_t>::addition_would_overflow(new_buffer_size, 1 * MiB)) + return false; + auto rounded_new_buffer_size_or_error = Memory::page_round_up(new_buffer_size + 1 * MiB); + if (rounded_new_buffer_size_or_error.is_error()) { + return false; + } + auto new_buffer_or_error = KBuffer::try_create_with_size("KBufferBuilder"sv, rounded_new_buffer_size_or_error.value()); + if (new_buffer_or_error.is_error()) + return false; + auto new_buffer = new_buffer_or_error.release_value(); + memcpy(new_buffer->data(), m_buffer->data(), m_buffer->size()); + m_buffer = move(new_buffer); + return true; +} + +bool KBufferBuilder::flush() +{ + if (!m_buffer) + return false; + m_buffer->set_size(m_size); + return true; +} + +OwnPtr<KBuffer> KBufferBuilder::build() +{ + if (!flush()) + return {}; + + return move(m_buffer); +} + +ErrorOr<KBufferBuilder> KBufferBuilder::try_create() +{ + auto buffer = TRY(KBuffer::try_create_with_size("KBufferBuilder"sv, 4 * MiB, Memory::Region::Access::ReadWrite)); + return KBufferBuilder { move(buffer) }; +} + +KBufferBuilder::KBufferBuilder(NonnullOwnPtr<KBuffer> buffer) + : m_buffer(move(buffer)) +{ +} + +ErrorOr<void> KBufferBuilder::append_bytes(ReadonlyBytes bytes) +{ + if (!check_expand(bytes.size())) + return ENOMEM; + memcpy(insertion_ptr(), bytes.data(), bytes.size()); + m_size += bytes.size(); + return {}; +} + +ErrorOr<void> KBufferBuilder::append(StringView str) +{ + if (str.is_empty()) + return {}; + if (!check_expand(str.length())) + return ENOMEM; + memcpy(insertion_ptr(), str.characters_without_null_termination(), str.length()); + m_size += str.length(); + return {}; +} + +ErrorOr<void> KBufferBuilder::append(char const* characters, int length) +{ + if (!length) + return {}; + if (!check_expand(length)) + return ENOMEM; + memcpy(insertion_ptr(), characters, length); + m_size += length; + return {}; +} + +ErrorOr<void> KBufferBuilder::append(char ch) +{ + if (!check_expand(1)) + return ENOMEM; + insertion_ptr()[0] = ch; + m_size += 1; + return {}; +} + +ErrorOr<void> KBufferBuilder::append_escaped_for_json(StringView string) +{ + for (auto ch : string) { + switch (ch) { + case '\b': + TRY(append("\\b"sv)); + break; + case '\n': + TRY(append("\\n"sv)); + break; + case '\t': + TRY(append("\\t"sv)); + break; + case '\"': + TRY(append("\\\""sv)); + break; + case '\\': + TRY(append("\\\\"sv)); + break; + default: + if (ch >= 0 && ch <= 0x1f) + TRY(appendff("\\u{:04x}", ch)); + else + TRY(append(ch)); + } + } + return {}; +} + +} diff --git a/Kernel/Library/KBufferBuilder.h b/Kernel/Library/KBufferBuilder.h new file mode 100644 index 0000000000..9b9a159d1f --- /dev/null +++ b/Kernel/Library/KBufferBuilder.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/StringBuilder.h> +#include <AK/StringView.h> +#include <Kernel/Library/KBuffer.h> + +namespace Kernel { + +class KBufferBuilder { + AK_MAKE_NONCOPYABLE(KBufferBuilder); + +public: + using OutputType = KBuffer; + + static ErrorOr<KBufferBuilder> try_create(); + + KBufferBuilder(KBufferBuilder&&) = default; + KBufferBuilder& operator=(KBufferBuilder&&) = default; + ~KBufferBuilder() = default; + + ErrorOr<void> append(StringView); + ErrorOr<void> append(char); + ErrorOr<void> append(char const*, int); + + ErrorOr<void> append_escaped_for_json(StringView); + ErrorOr<void> append_bytes(ReadonlyBytes); + + template<typename... Parameters> + ErrorOr<void> appendff(CheckedFormatString<Parameters...>&& fmtstr, Parameters const&... parameters) + { + // FIXME: This really not ideal, but vformat expects StringBuilder. + StringBuilder builder; + AK::VariadicFormatParams<AK::AllowDebugOnlyFormatters::No, Parameters...> variadic_format_params { parameters... }; + TRY(vformat(builder, fmtstr.view(), variadic_format_params)); + return append_bytes(builder.string_view().bytes()); + } + + bool flush(); + OwnPtr<KBuffer> build(); + + ReadonlyBytes bytes() const + { + if (!m_buffer) + return {}; + return m_buffer->bytes(); + } + + size_t length() const + { + return m_size; + } + +private: + explicit KBufferBuilder(NonnullOwnPtr<KBuffer>); + + bool check_expand(size_t); + u8* insertion_ptr() + { + if (!m_buffer) + return nullptr; + return m_buffer->data() + m_size; + } + + OwnPtr<KBuffer> m_buffer; + size_t m_size { 0 }; +}; + +} diff --git a/Kernel/Library/KLexicalPath.cpp b/Kernel/Library/KLexicalPath.cpp new file mode 100644 index 0000000000..011cd1c329 --- /dev/null +++ b/Kernel/Library/KLexicalPath.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021, Max Wipfli <max.wipfli@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Vector.h> +#include <Kernel/Library/KLexicalPath.h> + +namespace Kernel::KLexicalPath { + +static StringView const s_single_dot = "."sv; + +bool is_absolute(StringView path) +{ + return !path.is_empty() && path[0] == '/'; +} + +bool is_canonical(StringView path) +{ + // FIXME: This can probably be done more efficiently. + if (path.is_empty()) + return false; + if (path.ends_with('/') && path.length() != 1) + return false; + if (path.starts_with("./"sv) || path.contains("/./"sv) || path.ends_with("/."sv)) + return false; + if (path.starts_with("../"sv) || path.contains("/../"sv) || path.ends_with("/.."sv)) + return false; + if (path.contains("//"sv)) + return false; + return true; +} + +StringView basename(StringView a_path) +{ + if (a_path == "/"sv) + return a_path; + if (a_path.is_empty()) + return s_single_dot; + auto path = a_path.trim("/"sv, TrimMode::Right); + // NOTE: If it's empty now, it means the path was just a series of slashes. + if (path.is_empty()) + return a_path.substring_view(0, 1); + auto slash_index = path.find_last('/'); + if (!slash_index.has_value()) + return path; + auto basename = path.substring_view(*slash_index + 1); + return basename; +} + +StringView dirname(StringView path) +{ + VERIFY(is_canonical(path)); + auto slash_index = path.find_last('/'); + VERIFY(slash_index.has_value()); + return path.substring_view(0, *slash_index); +} + +Vector<StringView> parts(StringView path) +{ + VERIFY(is_canonical(path)); + return path.split_view('/'); +} + +ErrorOr<NonnullOwnPtr<KString>> try_join(StringView first, StringView second) +{ + VERIFY(is_canonical(first)); + VERIFY(is_canonical(second)); + VERIFY(!is_absolute(second)); + + if (first == "/"sv) { + char* buffer; + auto string = TRY(KString::try_create_uninitialized(1 + second.length(), buffer)); + buffer[0] = '/'; + __builtin_memcpy(buffer + 1, second.characters_without_null_termination(), second.length()); + buffer[string->length()] = 0; + return string; + } + char* buffer; + auto string = TRY(KString::try_create_uninitialized(first.length() + 1 + second.length(), buffer)); + __builtin_memcpy(buffer, first.characters_without_null_termination(), first.length()); + buffer[first.length()] = '/'; + __builtin_memcpy(buffer + first.length() + 1, second.characters_without_null_termination(), second.length()); + buffer[string->length()] = 0; + return string; +} + +} diff --git a/Kernel/Library/KLexicalPath.h b/Kernel/Library/KLexicalPath.h new file mode 100644 index 0000000000..abf7eb0591 --- /dev/null +++ b/Kernel/Library/KLexicalPath.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2021, Max Wipfli <max.wipfli@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/StringView.h> +#include <Kernel/Library/KString.h> + +namespace Kernel::KLexicalPath { + +bool is_absolute(StringView); +bool is_canonical(StringView); +StringView basename(StringView); +StringView dirname(StringView); +Vector<StringView> parts(StringView); + +ErrorOr<NonnullOwnPtr<KString>> try_join(StringView, StringView); + +} diff --git a/Kernel/Library/KString.cpp b/Kernel/Library/KString.cpp new file mode 100644 index 0000000000..8469de75a1 --- /dev/null +++ b/Kernel/Library/KString.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Format.h> +#include <AK/StringBuilder.h> +#include <Kernel/Library/KString.h> + +extern bool g_in_early_boot; + +namespace Kernel { + +ErrorOr<NonnullOwnPtr<KString>> KString::try_create(StringView string) +{ + char* characters = nullptr; + size_t length = string.length(); + auto new_string = TRY(KString::try_create_uninitialized(length, characters)); + if (!string.is_empty()) + __builtin_memcpy(characters, string.characters_without_null_termination(), length); + characters[length] = '\0'; + return new_string; +} + +ErrorOr<NonnullOwnPtr<KString>> KString::vformatted(StringView fmtstr, AK::TypeErasedFormatParams& params) +{ + StringBuilder builder; + TRY(AK::vformat(builder, fmtstr, params)); + return try_create(builder.string_view()); +} + +NonnullOwnPtr<KString> KString::must_create(StringView string) +{ + // We can only enforce success during early boot. + VERIFY(g_in_early_boot); + return KString::try_create(string).release_value(); +} + +ErrorOr<NonnullOwnPtr<KString>> KString::try_create_uninitialized(size_t length, char*& characters) +{ + size_t allocation_size = sizeof(KString) + (sizeof(char) * length) + sizeof(char); + auto* slot = kmalloc(allocation_size); + if (!slot) + return ENOMEM; + auto new_string = TRY(adopt_nonnull_own_or_enomem(new (slot) KString(length))); + characters = new_string->m_characters; + return new_string; +} + +NonnullOwnPtr<KString> KString::must_create_uninitialized(size_t length, char*& characters) +{ + // We can only enforce success during early boot. + VERIFY(g_in_early_boot); + return KString::try_create_uninitialized(length, characters).release_value(); +} + +ErrorOr<NonnullOwnPtr<KString>> KString::try_clone() const +{ + return try_create(view()); +} + +void KString::operator delete(void* string) +{ + if (!string) + return; + size_t allocation_size = sizeof(KString) + (sizeof(char) * static_cast<KString*>(string)->m_length) + sizeof(char); + kfree_sized(string, allocation_size); +} + +} diff --git a/Kernel/Library/KString.h b/Kernel/Library/KString.h new file mode 100644 index 0000000000..7905936dbe --- /dev/null +++ b/Kernel/Library/KString.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Format.h> +#include <AK/OwnPtr.h> + +namespace Kernel { + +class KString { + AK_MAKE_NONCOPYABLE(KString); + AK_MAKE_NONMOVABLE(KString); + +public: + [[nodiscard]] static ErrorOr<NonnullOwnPtr<KString>> try_create_uninitialized(size_t, char*&); + [[nodiscard]] static NonnullOwnPtr<KString> must_create_uninitialized(size_t, char*&); + [[nodiscard]] static ErrorOr<NonnullOwnPtr<KString>> try_create(StringView); + [[nodiscard]] static NonnullOwnPtr<KString> must_create(StringView); + + [[nodiscard]] static ErrorOr<NonnullOwnPtr<KString>> vformatted(StringView fmtstr, AK::TypeErasedFormatParams&); + + template<typename... Parameters> + [[nodiscard]] static ErrorOr<NonnullOwnPtr<KString>> formatted(CheckedFormatString<Parameters...>&& fmtstr, Parameters const&... parameters) + { + AK::VariadicFormatParams<AK::AllowDebugOnlyFormatters::No, Parameters...> variadic_format_parameters { parameters... }; + return vformatted(fmtstr.view(), variadic_format_parameters); + } + + [[nodiscard]] static ErrorOr<NonnullOwnPtr<KString>> number(Arithmetic auto value) + { + return formatted("{}", value); + } + + void operator delete(void*); + + ErrorOr<NonnullOwnPtr<KString>> try_clone() const; + + [[nodiscard]] bool is_empty() const { return m_length == 0; } + [[nodiscard]] size_t length() const { return m_length; } + [[nodiscard]] char const* characters() const { return m_characters; } + [[nodiscard]] StringView view() const { return { characters(), length() }; } + [[nodiscard]] ReadonlyBytes bytes() const { return { characters(), length() }; } + +private: + explicit KString(size_t length) + : m_length(length) + { + } + + size_t m_length { 0 }; + char m_characters[0]; +}; + +} + +namespace AK { + +template<> +struct Formatter<Kernel::KString> : Formatter<StringView> { + ErrorOr<void> format(FormatBuilder& builder, Kernel::KString const& value) + { + return Formatter<StringView>::format(builder, value.view()); + } +}; + +template<> +struct Formatter<OwnPtr<Kernel::KString>> : Formatter<StringView> { + ErrorOr<void> format(FormatBuilder& builder, OwnPtr<Kernel::KString> const& value) + { + if (value) + return Formatter<StringView>::format(builder, value->view()); + return Formatter<StringView>::format(builder, "[out of memory]"sv); + } +}; + +template<> +struct Formatter<NonnullOwnPtr<Kernel::KString>> : Formatter<StringView> { + ErrorOr<void> format(FormatBuilder& builder, NonnullOwnPtr<Kernel::KString> const& value) + { + return Formatter<StringView>::format(builder, value->view()); + } +}; + +template<> +struct Traits<NonnullOwnPtr<Kernel::KString>> : public GenericTraits<NonnullOwnPtr<Kernel::KString>> { + using PeekType = Kernel::KString*; + using ConstPeekType = Kernel::KString const*; + static unsigned hash(NonnullOwnPtr<Kernel::KString> const& p) { return string_hash(p->characters(), p->length()); } + static bool equals(NonnullOwnPtr<Kernel::KString> const& a, NonnullOwnPtr<Kernel::KString> const& b) { return a->view() == b->view(); } + static bool equals(StringView a, NonnullOwnPtr<Kernel::KString> const& b) { return a == b->view(); } +}; + +template<> +struct Traits<OwnPtr<Kernel::KString>> : public GenericTraits<OwnPtr<Kernel::KString>> { + using PeekType = Kernel::KString*; + using ConstPeekType = Kernel::KString const*; + static unsigned hash(OwnPtr<Kernel::KString> const& p) + { + if (!p) + return ptr_hash(nullptr); + return string_hash(p->characters(), p->length()); + } + static bool equals(OwnPtr<Kernel::KString> const& a, OwnPtr<Kernel::KString> const& b) + { + if (!a || !b) + return a.ptr() == b.ptr(); + if (a == b) + return true; + + return a->view() == b->view(); + } + static bool equals(StringView a, OwnPtr<Kernel::KString> const& b) + { + if (!b) + return a.is_null(); + return a == b->view(); + } +}; + +namespace Detail { +template<> +inline constexpr bool IsHashCompatible<StringView, NonnullOwnPtr<Kernel::KString>> = true; +template<> +inline constexpr bool IsHashCompatible<StringView, OwnPtr<Kernel::KString>> = true; +} + +} diff --git a/Kernel/Library/LockRefPtr.h b/Kernel/Library/LockRefPtr.h index 819acfd874..6486b0fa7c 100644 --- a/Kernel/Library/LockRefPtr.h +++ b/Kernel/Library/LockRefPtr.h @@ -16,7 +16,7 @@ #include <Kernel/Library/NonnullLockRefPtr.h> #ifdef KERNEL # include <Kernel/Arch/Processor.h> -# include <Kernel/ScopedCritical.h> +# include <Kernel/Library/ScopedCritical.h> #endif #define LOCKREFPTR_SCRUB_BYTE 0xa0 diff --git a/Kernel/Library/LockWeakable.h b/Kernel/Library/LockWeakable.h index 6c146e665b..0869ae4018 100644 --- a/Kernel/Library/LockWeakable.h +++ b/Kernel/Library/LockWeakable.h @@ -12,7 +12,7 @@ #include <AK/StdLibExtras.h> #include <Kernel/Arch/Processor.h> #include <Kernel/Library/LockRefPtr.h> -#include <Kernel/ScopedCritical.h> +#include <Kernel/Library/ScopedCritical.h> namespace AK { diff --git a/Kernel/Library/MiniStdLib.cpp b/Kernel/Library/MiniStdLib.cpp new file mode 100644 index 0000000000..023f9f1155 --- /dev/null +++ b/Kernel/Library/MiniStdLib.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Library/StdLib.h> + +extern "C" { + +void* memcpy(void* dest_ptr, void const* src_ptr, size_t n) +{ +#if ARCH(X86_64) + size_t dest = (size_t)dest_ptr; + size_t src = (size_t)src_ptr; + // FIXME: Support starting at an unaligned address. + if (!(dest & 0x3) && !(src & 0x3) && n >= 12) { + size_t size_ts = n / sizeof(size_t); + asm volatile( + "rep movsq\n" + : "=S"(src), "=D"(dest) + : "S"(src), "D"(dest), "c"(size_ts) + : "memory"); + n -= size_ts * sizeof(size_t); + if (n == 0) + return dest_ptr; + } + asm volatile( + "rep movsb\n" ::"S"(src), "D"(dest), "c"(n) + : "memory"); +#else + u8* pd = (u8*)dest_ptr; + u8 const* ps = (u8 const*)src_ptr; + for (; n--;) + *pd++ = *ps++; +#endif + return dest_ptr; +} + +void* memmove(void* dest, void const* src, size_t n) +{ + if (dest < src) + return memcpy(dest, src, n); + + u8* pd = (u8*)dest; + u8 const* ps = (u8 const*)src; + for (pd += n, ps += n; n--;) + *--pd = *--ps; + return dest; +} + +void* memset(void* dest_ptr, int c, size_t n) +{ +#if ARCH(X86_64) + size_t dest = (size_t)dest_ptr; + // FIXME: Support starting at an unaligned address. + if (!(dest & 0x3) && n >= 12) { + size_t size_ts = n / sizeof(size_t); + size_t expanded_c = explode_byte((u8)c); + asm volatile( + "rep stosq\n" + : "=D"(dest) + : "D"(dest), "c"(size_ts), "a"(expanded_c) + : "memory"); + n -= size_ts * sizeof(size_t); + if (n == 0) + return dest_ptr; + } + asm volatile( + "rep stosb\n" + : "=D"(dest), "=c"(n) + : "0"(dest), "1"(n), "a"(c) + : "memory"); +#else + u8* pd = (u8*)dest_ptr; + for (; n--;) + *pd++ = c; +#endif + return dest_ptr; +} + +size_t strlen(char const* str) +{ + size_t len = 0; + while (*(str++)) + ++len; + return len; +} + +size_t strnlen(char const* str, size_t maxlen) +{ + size_t len = 0; + for (; len < maxlen && *str; str++) + len++; + return len; +} + +int strcmp(char const* s1, char const* s2) +{ + for (; *s1 == *s2; ++s1, ++s2) { + if (*s1 == 0) + return 0; + } + return *(u8 const*)s1 < *(u8 const*)s2 ? -1 : 1; +} + +int memcmp(void const* v1, void const* v2, size_t n) +{ + auto const* s1 = (u8 const*)v1; + auto const* s2 = (u8 const*)v2; + while (n-- > 0) { + if (*s1++ != *s2++) + return s1[-1] < s2[-1] ? -1 : 1; + } + return 0; +} + +int strncmp(char const* s1, char const* s2, size_t n) +{ + if (!n) + return 0; + do { + if (*s1 != *s2++) + return *(unsigned char const*)s1 - *(unsigned char const*)--s2; + if (*s1++ == 0) + break; + } while (--n); + return 0; +} + +char* strstr(char const* haystack, char const* needle) +{ + char nch; + char hch; + + if ((nch = *needle++) != 0) { + size_t len = strlen(needle); + do { + do { + if ((hch = *haystack++) == 0) + return nullptr; + } while (hch != nch); + } while (strncmp(haystack, needle, len) != 0); + --haystack; + } + return const_cast<char*>(haystack); +} +} diff --git a/Kernel/Library/NonnullLockRefPtr.h b/Kernel/Library/NonnullLockRefPtr.h index 643fec04de..3b915692fa 100644 --- a/Kernel/Library/NonnullLockRefPtr.h +++ b/Kernel/Library/NonnullLockRefPtr.h @@ -14,7 +14,7 @@ #include <AK/Types.h> #ifdef KERNEL # include <Kernel/Arch/Processor.h> -# include <Kernel/ScopedCritical.h> +# include <Kernel/Library/ScopedCritical.h> #endif #define NONNULLLOCKREFPTR_SCRUB_BYTE 0xa1 diff --git a/Kernel/Library/Panic.cpp b/Kernel/Library/Panic.cpp new file mode 100644 index 0000000000..227cc1b22a --- /dev/null +++ b/Kernel/Library/Panic.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Format.h> +#include <Kernel/Arch/Processor.h> +#if ARCH(X86_64) +# include <Kernel/Arch/x86_64/Shutdown.h> +#elif ARCH(AARCH64) +# include <Kernel/Arch/aarch64/RPi/Watchdog.h> +#endif +#include <Kernel/CommandLine.h> +#include <Kernel/KSyms.h> +#include <Kernel/Library/Panic.h> +#include <Kernel/Tasks/Thread.h> + +namespace Kernel { + +[[noreturn]] static void __shutdown() +{ +#if ARCH(X86_64) + qemu_shutdown(); + virtualbox_shutdown(); +#elif ARCH(AARCH64) + RPi::Watchdog::the().system_shutdown(); +#endif + // Note: If we failed to invoke platform shutdown, we need to halt afterwards + // to ensure no further execution on any CPU still happens. + Processor::halt(); +} + +void __panic(char const* file, unsigned int line, char const* function) +{ + // Avoid lock ranking checks on crashing paths, just try to get some debugging messages out. + auto* thread = Thread::current(); + if (thread) + thread->set_crashing(); + + critical_dmesgln("at {}:{} in {}", file, line, function); + dump_backtrace(PrintToScreen::Yes); + if (!CommandLine::was_initialized()) + Processor::halt(); + switch (kernel_command_line().panic_mode()) { + case PanicMode::Shutdown: + __shutdown(); + case PanicMode::Halt: + [[fallthrough]]; + default: + Processor::halt(); + } +} +} diff --git a/Kernel/Library/Panic.h b/Kernel/Library/Panic.h new file mode 100644 index 0000000000..0829964026 --- /dev/null +++ b/Kernel/Library/Panic.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Format.h> + +namespace Kernel { + +[[noreturn]] void __panic(char const* file, unsigned int line, char const* function); + +#define PANIC(...) \ + do { \ + critical_dmesgln("KERNEL PANIC! :^("); \ + critical_dmesgln(__VA_ARGS__); \ + __panic(__FILE__, __LINE__, __PRETTY_FUNCTION__); \ + } while (0) + +} diff --git a/Kernel/Library/ScopedCritical.cpp b/Kernel/Library/ScopedCritical.cpp new file mode 100644 index 0000000000..164a036233 --- /dev/null +++ b/Kernel/Library/ScopedCritical.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Library/ScopedCritical.h> + +#include <Kernel/Arch/Processor.h> + +namespace Kernel { + +ScopedCritical::ScopedCritical() +{ + enter(); +} + +ScopedCritical::~ScopedCritical() +{ + if (m_valid) + leave(); +} + +ScopedCritical::ScopedCritical(ScopedCritical&& from) + : m_valid(exchange(from.m_valid, false)) +{ +} + +ScopedCritical& ScopedCritical::operator=(ScopedCritical&& from) +{ + if (&from != this) { + m_valid = exchange(from.m_valid, false); + } + return *this; +} + +void ScopedCritical::leave() +{ + VERIFY(m_valid); + m_valid = false; + Processor::leave_critical(); +} + +void ScopedCritical::enter() +{ + VERIFY(!m_valid); + m_valid = true; + Processor::enter_critical(); +} + +} diff --git a/Kernel/Library/ScopedCritical.h b/Kernel/Library/ScopedCritical.h new file mode 100644 index 0000000000..678069e4f6 --- /dev/null +++ b/Kernel/Library/ScopedCritical.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Noncopyable.h> + +namespace Kernel { + +class ScopedCritical { + AK_MAKE_NONCOPYABLE(ScopedCritical); + +public: + ScopedCritical(); + ~ScopedCritical(); + ScopedCritical(ScopedCritical&& from); + + ScopedCritical& operator=(ScopedCritical&& from); + + void leave(); + void enter(); + +private: + bool m_valid { false }; +}; + +} diff --git a/Kernel/Library/StdLib.cpp b/Kernel/Library/StdLib.cpp new file mode 100644 index 0000000000..8f003672db --- /dev/null +++ b/Kernel/Library/StdLib.cpp @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Assertions.h> +#include <AK/MemMem.h> +#include <AK/Types.h> +#include <Kernel/Arch/SafeMem.h> +#include <Kernel/Arch/SmapDisabler.h> +#include <Kernel/Library/StdLib.h> +#include <Kernel/Memory/MemoryManager.h> + +ErrorOr<NonnullOwnPtr<Kernel::KString>> try_copy_kstring_from_user(Userspace<char const*> user_str, size_t user_str_size) +{ + bool is_user = Kernel::Memory::is_user_range(user_str.vaddr(), user_str_size); + if (!is_user) + return EFAULT; + Kernel::SmapDisabler disabler; + void* fault_at; + ssize_t length = Kernel::safe_strnlen(user_str.unsafe_userspace_ptr(), user_str_size, fault_at); + if (length < 0) { + dbgln("copy_kstring_from_user({:p}, {}) failed at {} (strnlen)", static_cast<void const*>(user_str.unsafe_userspace_ptr()), user_str_size, VirtualAddress { fault_at }); + return EFAULT; + } + char* buffer; + auto new_string = TRY(Kernel::KString::try_create_uninitialized(length, buffer)); + + buffer[length] = '\0'; + + if (length == 0) + return new_string; + + if (!Kernel::safe_memcpy(buffer, user_str.unsafe_userspace_ptr(), (size_t)length, fault_at)) { + dbgln("copy_kstring_from_user({:p}, {}) failed at {} (memcpy)", static_cast<void const*>(user_str.unsafe_userspace_ptr()), user_str_size, VirtualAddress { fault_at }); + return EFAULT; + } + return new_string; +} + +ErrorOr<Duration> copy_time_from_user(timespec const* ts_user) +{ + timespec ts {}; + TRY(copy_from_user(&ts, ts_user, sizeof(timespec))); + return Duration::from_timespec(ts); +} + +ErrorOr<Duration> copy_time_from_user(timeval const* tv_user) +{ + timeval tv {}; + TRY(copy_from_user(&tv, tv_user, sizeof(timeval))); + return Duration::from_timeval(tv); +} + +template<> +ErrorOr<Duration> copy_time_from_user<timeval const>(Userspace<timeval const*> src) { return copy_time_from_user(src.unsafe_userspace_ptr()); } +template<> +ErrorOr<Duration> copy_time_from_user<timeval>(Userspace<timeval*> src) { return copy_time_from_user(src.unsafe_userspace_ptr()); } +template<> +ErrorOr<Duration> copy_time_from_user<timespec const>(Userspace<timespec const*> src) { return copy_time_from_user(src.unsafe_userspace_ptr()); } +template<> +ErrorOr<Duration> copy_time_from_user<timespec>(Userspace<timespec*> src) { return copy_time_from_user(src.unsafe_userspace_ptr()); } + +Optional<u32> user_atomic_fetch_add_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_fetch_add_relaxed(var, val); +} + +Optional<u32> user_atomic_exchange_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_exchange_relaxed(var, val); +} + +Optional<u32> user_atomic_load_relaxed(u32 volatile* var) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_load_relaxed(var); +} + +bool user_atomic_store_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return false; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return false; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_store_relaxed(var, val); +} + +Optional<bool> user_atomic_compare_exchange_relaxed(u32 volatile* var, u32& expected, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + VERIFY(!Kernel::Memory::is_user_range(VirtualAddress(&expected), sizeof(expected))); + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_compare_exchange_relaxed(var, expected, val); +} + +Optional<u32> user_atomic_fetch_and_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_fetch_and_relaxed(var, val); +} + +Optional<u32> user_atomic_fetch_and_not_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_fetch_and_not_relaxed(var, val); +} + +Optional<u32> user_atomic_fetch_or_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_fetch_or_relaxed(var, val); +} + +Optional<u32> user_atomic_fetch_xor_relaxed(u32 volatile* var, u32 val) +{ + if (FlatPtr(var) & 3) + return {}; // not aligned! + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(FlatPtr(var)), sizeof(*var)); + if (!is_user) + return {}; + Kernel::SmapDisabler disabler; + return Kernel::safe_atomic_fetch_xor_relaxed(var, val); +} + +ErrorOr<void> copy_to_user(void* dest_ptr, void const* src_ptr, size_t n) +{ + if (!Kernel::Memory::is_user_range(VirtualAddress(dest_ptr), n)) + return EFAULT; + VERIFY(!Kernel::Memory::is_user_range(VirtualAddress(src_ptr), n)); + Kernel::SmapDisabler disabler; + void* fault_at; + if (!Kernel::safe_memcpy(dest_ptr, src_ptr, n, fault_at)) { + VERIFY(VirtualAddress(fault_at) >= VirtualAddress(dest_ptr) && VirtualAddress(fault_at) <= VirtualAddress((FlatPtr)dest_ptr + n)); + dbgln("copy_to_user({:p}, {:p}, {}) failed at {}", dest_ptr, src_ptr, n, VirtualAddress { fault_at }); + return EFAULT; + } + return {}; +} + +ErrorOr<void> copy_from_user(void* dest_ptr, void const* src_ptr, size_t n) +{ + if (!Kernel::Memory::is_user_range(VirtualAddress(src_ptr), n)) + return EFAULT; + VERIFY(!Kernel::Memory::is_user_range(VirtualAddress(dest_ptr), n)); + Kernel::SmapDisabler disabler; + void* fault_at; + if (!Kernel::safe_memcpy(dest_ptr, src_ptr, n, fault_at)) { + VERIFY(VirtualAddress(fault_at) >= VirtualAddress(src_ptr) && VirtualAddress(fault_at) <= VirtualAddress((FlatPtr)src_ptr + n)); + dbgln("copy_from_user({:p}, {:p}, {}) failed at {}", dest_ptr, src_ptr, n, VirtualAddress { fault_at }); + return EFAULT; + } + return {}; +} + +ErrorOr<void> memset_user(void* dest_ptr, int c, size_t n) +{ + bool is_user = Kernel::Memory::is_user_range(VirtualAddress(dest_ptr), n); + if (!is_user) + return EFAULT; + Kernel::SmapDisabler disabler; + void* fault_at; + if (!Kernel::safe_memset(dest_ptr, c, n, fault_at)) { + dbgln("memset_user({:p}, {}, {}) failed at {}", dest_ptr, c, n, VirtualAddress { fault_at }); + return EFAULT; + } + return {}; +} + +#if defined(AK_COMPILER_CLANG) && defined(ENABLE_KERNEL_LTO) +// Due to a chicken-and-egg situation, certain linker-defined symbols that are added on-demand (like the GOT) +// need to be present before LTO bitcode files are compiled. And since we don't link to any native object files, +// the linker does not know that _GLOBAL_OFFSET_TABLE_ is needed, so it doesn't define it, so linking as a PIE fails. +// See https://bugs.llvm.org/show_bug.cgi?id=39634 +FlatPtr missing_got_workaround() +{ + extern volatile FlatPtr _GLOBAL_OFFSET_TABLE_; + return _GLOBAL_OFFSET_TABLE_; +} +#endif + +extern "C" { + +void const* memmem(void const* haystack, size_t haystack_length, void const* needle, size_t needle_length) +{ + return AK::memmem(haystack, haystack_length, needle, needle_length); +} + +// Functions that are automatically called by the C++ compiler. +// Declare them first, to tell the silly compiler that they are indeed being used. +[[noreturn]] void __stack_chk_fail() __attribute__((used)); +[[noreturn]] void __stack_chk_fail_local() __attribute__((used)); +extern "C" int __cxa_atexit(void (*)(void*), void*, void*); +[[noreturn]] void __cxa_pure_virtual(); + +[[noreturn]] void __stack_chk_fail() +{ + VERIFY_NOT_REACHED(); +} + +[[noreturn]] void __stack_chk_fail_local() +{ + VERIFY_NOT_REACHED(); +} + +extern "C" int __cxa_atexit(void (*)(void*), void*, void*) +{ + VERIFY_NOT_REACHED(); + return 0; +} + +[[noreturn]] void __cxa_pure_virtual() +{ + VERIFY_NOT_REACHED(); +} +} diff --git a/Kernel/Library/StdLib.h b/Kernel/Library/StdLib.h new file mode 100644 index 0000000000..ead5b06c60 --- /dev/null +++ b/Kernel/Library/StdLib.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Checked.h> +#include <AK/Error.h> +#include <AK/Forward.h> +#include <AK/Time.h> +#include <AK/Userspace.h> +#include <Kernel/Library/KString.h> +#include <Kernel/UnixTypes.h> + +ErrorOr<NonnullOwnPtr<Kernel::KString>> try_copy_kstring_from_user(Userspace<char const*>, size_t); +ErrorOr<Duration> copy_time_from_user(timespec const*); +ErrorOr<Duration> copy_time_from_user(timeval const*); +template<typename T> +ErrorOr<Duration> copy_time_from_user(Userspace<T*>); + +[[nodiscard]] Optional<u32> user_atomic_fetch_add_relaxed(u32 volatile* var, u32 val); +[[nodiscard]] Optional<u32> user_atomic_exchange_relaxed(u32 volatile* var, u32 val); +[[nodiscard]] Optional<u32> user_atomic_load_relaxed(u32 volatile* var); +[[nodiscard]] bool user_atomic_store_relaxed(u32 volatile* var, u32 val); +[[nodiscard]] Optional<bool> user_atomic_compare_exchange_relaxed(u32 volatile* var, u32& expected, u32 val); +[[nodiscard]] Optional<u32> user_atomic_fetch_and_relaxed(u32 volatile* var, u32 val); +[[nodiscard]] Optional<u32> user_atomic_fetch_and_not_relaxed(u32 volatile* var, u32 val); +[[nodiscard]] Optional<u32> user_atomic_fetch_or_relaxed(u32 volatile* var, u32 val); +[[nodiscard]] Optional<u32> user_atomic_fetch_xor_relaxed(u32 volatile* var, u32 val); + +ErrorOr<void> copy_to_user(void*, void const*, size_t); +ErrorOr<void> copy_from_user(void*, void const*, size_t); +ErrorOr<void> memset_user(void*, int, size_t); + +extern "C" { + +void* memcpy(void*, void const*, size_t); +[[nodiscard]] int strncmp(char const* s1, char const* s2, size_t n); +[[nodiscard]] char* strstr(char const* haystack, char const* needle); +[[nodiscard]] int strcmp(char const*, char const*); +[[nodiscard]] size_t strlen(char const*); +[[nodiscard]] size_t strnlen(char const*, size_t); +void* memset(void*, int, size_t); +[[nodiscard]] int memcmp(void const*, void const*, size_t); +void* memmove(void* dest, void const* src, size_t n); +void const* memmem(void const* haystack, size_t, void const* needle, size_t); + +[[nodiscard]] inline u16 ntohs(u16 w) { return (w & 0xff) << 8 | ((w >> 8) & 0xff); } +[[nodiscard]] inline u16 htons(u16 w) { return (w & 0xff) << 8 | ((w >> 8) & 0xff); } +} + +#define offsetof(type, member) __builtin_offsetof(type, member) + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_from_user(T* dest, T const* src) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_from_user(dest, src, sizeof(T)); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_to_user(T* dest, T const* src) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_to_user(dest, src, sizeof(T)); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_from_user(T* dest, Userspace<T const*> src) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_from_user(dest, src.unsafe_userspace_ptr(), sizeof(T)); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_from_user(T* dest, Userspace<T*> src) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_from_user(dest, src.unsafe_userspace_ptr(), sizeof(T)); +} + +#define DEPRECATE_COPY_FROM_USER_TYPE(T, REPLACEMENT) \ + template<> \ + [[nodiscard]] inline __attribute__((deprecated("use " #REPLACEMENT " instead"))) ErrorOr<void> copy_from_user<T>(T*, const T*) \ + { \ + VERIFY_NOT_REACHED(); \ + } \ + template<> \ + [[nodiscard]] inline __attribute__((deprecated("use " #REPLACEMENT " instead"))) ErrorOr<void> copy_from_user<T>(T*, Userspace<const T*>) \ + { \ + VERIFY_NOT_REACHED(); \ + } \ + template<> \ + [[nodiscard]] inline __attribute__((deprecated("use " #REPLACEMENT " instead"))) ErrorOr<void> copy_from_user<T>(T*, Userspace<T*>) \ + { \ + VERIFY_NOT_REACHED(); \ + } + +DEPRECATE_COPY_FROM_USER_TYPE(timespec, copy_time_from_user) +DEPRECATE_COPY_FROM_USER_TYPE(timeval, copy_time_from_user) + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_to_user(Userspace<T*> dest, T const* src) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_to_user(dest.unsafe_userspace_ptr(), src, sizeof(T)); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_to_user(Userspace<T*> dest, void const* src, size_t size) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_to_user(dest.unsafe_userspace_ptr(), src, size); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_from_user(void* dest, Userspace<T const*> src, size_t size) +{ + static_assert(IsTriviallyCopyable<T>); + return copy_from_user(dest, src.unsafe_userspace_ptr(), size); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_n_from_user(T* dest, T const* src, size_t count) +{ + static_assert(IsTriviallyCopyable<T>); + Checked<size_t> size = sizeof(T); + size *= count; + if (size.has_overflow()) + return EOVERFLOW; + return copy_from_user(dest, src, size.value()); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_n_to_user(T* dest, T const* src, size_t count) +{ + static_assert(IsTriviallyCopyable<T>); + Checked<size_t> size = sizeof(T); + size *= count; + if (size.has_overflow()) + return EOVERFLOW; + return copy_to_user(dest, src, size.value()); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_n_from_user(T* dest, Userspace<T const*> src, size_t count) +{ + static_assert(IsTriviallyCopyable<T>); + Checked<size_t> size = sizeof(T); + size *= count; + if (size.has_overflow()) + return EOVERFLOW; + return copy_from_user(dest, src.unsafe_userspace_ptr(), size.value()); +} + +template<typename T> +[[nodiscard]] inline ErrorOr<void> copy_n_to_user(Userspace<T*> dest, T const* src, size_t count) +{ + static_assert(IsTriviallyCopyable<T>); + Checked<size_t> size = sizeof(T); + size *= count; + if (size.has_overflow()) + return EOVERFLOW; + return copy_to_user(dest.unsafe_userspace_ptr(), src, size.value()); +} + +template<typename T> +inline ErrorOr<T> copy_typed_from_user(Userspace<T const*> user_data) +{ + T data {}; + TRY(copy_from_user(&data, user_data)); + return data; +} + +template<typename T> +inline ErrorOr<T> copy_typed_from_user(Userspace<T*> user_data) +{ + T data {}; + TRY(copy_from_user(&data, user_data)); + return data; +} diff --git a/Kernel/Library/UserOrKernelBuffer.cpp b/Kernel/Library/UserOrKernelBuffer.cpp new file mode 100644 index 0000000000..0dcefa5a47 --- /dev/null +++ b/Kernel/Library/UserOrKernelBuffer.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2020, the SerenityOS developers. + * Copyright (c) 2020, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Library/UserOrKernelBuffer.h> +#include <Kernel/Memory/MemoryManager.h> + +namespace Kernel { + +bool UserOrKernelBuffer::is_kernel_buffer() const +{ + return !Memory::is_user_address(VirtualAddress(m_buffer)); +} + +ErrorOr<NonnullOwnPtr<KString>> UserOrKernelBuffer::try_copy_into_kstring(size_t size) const +{ + if (!m_buffer) + return EINVAL; + if (Memory::is_user_address(VirtualAddress(m_buffer))) { + char* buffer; + auto kstring = TRY(KString::try_create_uninitialized(size, buffer)); + TRY(copy_from_user(buffer, m_buffer, size)); + return kstring; + } + + return KString::try_create(ReadonlyBytes { m_buffer, size }); +} + +ErrorOr<void> UserOrKernelBuffer::write(void const* src, size_t offset, size_t len) +{ + if (!m_buffer) + return EFAULT; + + if (Memory::is_user_address(VirtualAddress(m_buffer))) + return copy_to_user(m_buffer + offset, src, len); + + memcpy(m_buffer + offset, src, len); + return {}; +} + +ErrorOr<void> UserOrKernelBuffer::read(void* dest, size_t offset, size_t len) const +{ + if (!m_buffer) + return EFAULT; + + if (Memory::is_user_address(VirtualAddress(m_buffer))) + return copy_from_user(dest, m_buffer + offset, len); + + memcpy(dest, m_buffer + offset, len); + return {}; +} + +ErrorOr<void> UserOrKernelBuffer::memset(int value, size_t offset, size_t len) +{ + if (!m_buffer) + return EFAULT; + + if (Memory::is_user_address(VirtualAddress(m_buffer))) + return memset_user(m_buffer + offset, value, len); + + ::memset(m_buffer + offset, value, len); + return {}; +} + +} diff --git a/Kernel/Library/UserOrKernelBuffer.h b/Kernel/Library/UserOrKernelBuffer.h new file mode 100644 index 0000000000..41f9fd4dad --- /dev/null +++ b/Kernel/Library/UserOrKernelBuffer.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2020, the SerenityOS developers. + * Copyright (c) 2021, Andreas Kling <kling@serenityos.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> +#include <AK/Userspace.h> +#include <Kernel/API/POSIX/errno.h> +#include <Kernel/Library/StdLib.h> +#include <Kernel/Memory/MemoryManager.h> +#include <Kernel/UnixTypes.h> + +namespace Kernel { + +class [[nodiscard]] UserOrKernelBuffer { +public: + UserOrKernelBuffer() = delete; + + static UserOrKernelBuffer for_kernel_buffer(u8* kernel_buffer) + { + VERIFY(!kernel_buffer || !Memory::is_user_address(VirtualAddress(kernel_buffer))); + return UserOrKernelBuffer(kernel_buffer); + } + + static ErrorOr<UserOrKernelBuffer> for_user_buffer(u8* user_buffer, size_t size) + { + if (user_buffer && !Memory::is_user_range(VirtualAddress(user_buffer), size)) + return Error::from_errno(EFAULT); + return UserOrKernelBuffer(user_buffer); + } + + template<typename UserspaceType> + static ErrorOr<UserOrKernelBuffer> for_user_buffer(UserspaceType userspace, size_t size) + { + if (!Memory::is_user_range(VirtualAddress(userspace.unsafe_userspace_ptr()), size)) + return Error::from_errno(EFAULT); + return UserOrKernelBuffer(const_cast<u8*>((u8 const*)userspace.unsafe_userspace_ptr())); + } + + [[nodiscard]] bool is_kernel_buffer() const; + [[nodiscard]] void const* user_or_kernel_ptr() const { return m_buffer; } + + [[nodiscard]] UserOrKernelBuffer offset(size_t offset) const + { + if (!m_buffer) + return *this; + UserOrKernelBuffer offset_buffer = *this; + offset_buffer.m_buffer += offset; + VERIFY(offset_buffer.is_kernel_buffer() == is_kernel_buffer()); + return offset_buffer; + } + + ErrorOr<NonnullOwnPtr<KString>> try_copy_into_kstring(size_t) const; + ErrorOr<void> write(void const* src, size_t offset, size_t len); + ErrorOr<void> write(void const* src, size_t len) + { + return write(src, 0, len); + } + ErrorOr<void> write(ReadonlyBytes bytes) + { + return write(bytes.data(), bytes.size()); + } + + ErrorOr<void> read(void* dest, size_t offset, size_t len) const; + ErrorOr<void> read(void* dest, size_t len) const + { + return read(dest, 0, len); + } + + ErrorOr<void> read(Bytes bytes) const + { + return read(bytes.data(), bytes.size()); + } + + ErrorOr<void> memset(int value, size_t offset, size_t len); + ErrorOr<void> memset(int value, size_t len) + { + return memset(value, 0, len); + } + + template<size_t BUFFER_BYTES, typename F> + ErrorOr<size_t> write_buffered(size_t offset, size_t len, F f) + { + if (!m_buffer) + return EFAULT; + if (is_kernel_buffer()) { + // We're transferring directly to a kernel buffer, bypass + Bytes bytes { m_buffer + offset, len }; + return f(bytes); + } + + // The purpose of using a buffer on the stack is that we can + // avoid a bunch of small (e.g. 1-byte) copy_to_user calls + u8 buffer[BUFFER_BYTES]; + size_t nwritten = 0; + while (nwritten < len) { + auto to_copy = min(sizeof(buffer), len - nwritten); + Bytes bytes { buffer, to_copy }; + ErrorOr<size_t> copied_or_error = f(bytes); + if (copied_or_error.is_error()) + return copied_or_error.release_error(); + auto copied = copied_or_error.release_value(); + VERIFY(copied <= to_copy); + TRY(write(buffer, nwritten, copied)); + nwritten += copied; + if (copied < to_copy) + break; + } + return nwritten; + } + template<size_t BUFFER_BYTES, typename F> + ErrorOr<size_t> write_buffered(size_t len, F f) + { + return write_buffered<BUFFER_BYTES, F>(0, len, f); + } + + template<size_t BUFFER_BYTES, typename F> + ErrorOr<size_t> read_buffered(size_t offset, size_t len, F f) const + { + if (!m_buffer) + return EFAULT; + if (is_kernel_buffer()) { + // We're transferring directly from a kernel buffer, bypass + return f({ m_buffer + offset, len }); + } + + // The purpose of using a buffer on the stack is that we can + // avoid a bunch of small (e.g. 1-byte) copy_from_user calls + u8 buffer[BUFFER_BYTES]; + size_t nread = 0; + while (nread < len) { + auto to_copy = min(sizeof(buffer), len - nread); + TRY(read(buffer, nread, to_copy)); + ReadonlyBytes read_only_bytes { buffer, to_copy }; + ErrorOr<size_t> copied_or_error = f(read_only_bytes); + if (copied_or_error.is_error()) + return copied_or_error.release_error(); + auto copied = copied_or_error.release_value(); + VERIFY(copied <= to_copy); + nread += copied; + if (copied < to_copy) + break; + } + return nread; + } + template<size_t BUFFER_BYTES, typename F> + ErrorOr<size_t> read_buffered(size_t len, F f) const + { + return read_buffered<BUFFER_BYTES, F>(0, len, f); + } + +private: + explicit UserOrKernelBuffer(u8* buffer) + : m_buffer(buffer) + { + } + + u8* m_buffer; +}; + +} |