/* * Copyright (c) 2018-2020, Andreas Kling * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once #include #include #include #include #include #include #ifdef KERNEL # include # include #endif #define NONNULLLOCKREFPTR_SCRUB_BYTE 0xa1 namespace AK { template class OwnPtr; template class LockRefPtr; template class [[nodiscard]] NonnullLockRefPtr { template friend class LockRefPtr; template friend class NonnullLockRefPtr; template friend class LockWeakPtr; public: using ElementType = T; enum AdoptTag { Adopt }; ALWAYS_INLINE NonnullLockRefPtr(T const& object) : m_bits((FlatPtr)&object) { VERIFY(!(m_bits & 1)); const_cast(object).ref(); } template ALWAYS_INLINE NonnullLockRefPtr(U const& object) requires(IsConvertible) : m_bits((FlatPtr) static_cast(&object)) { VERIFY(!(m_bits & 1)); const_cast(static_cast(object)).ref(); } ALWAYS_INLINE NonnullLockRefPtr(AdoptTag, T& object) : m_bits((FlatPtr)&object) { VERIFY(!(m_bits & 1)); } ALWAYS_INLINE NonnullLockRefPtr(NonnullLockRefPtr&& other) : m_bits((FlatPtr)&other.leak_ref()) { VERIFY(!(m_bits & 1)); } template ALWAYS_INLINE NonnullLockRefPtr(NonnullLockRefPtr&& other) requires(IsConvertible) : m_bits((FlatPtr)&other.leak_ref()) { VERIFY(!(m_bits & 1)); } ALWAYS_INLINE NonnullLockRefPtr(NonnullLockRefPtr const& other) : m_bits((FlatPtr)other.add_ref()) { VERIFY(!(m_bits & 1)); } template ALWAYS_INLINE NonnullLockRefPtr(NonnullLockRefPtr const& other) requires(IsConvertible) : m_bits((FlatPtr)other.add_ref()) { VERIFY(!(m_bits & 1)); } ALWAYS_INLINE ~NonnullLockRefPtr() { assign(nullptr); #ifdef SANITIZE_PTRS m_bits.store(explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE), AK::MemoryOrder::memory_order_relaxed); #endif } template NonnullLockRefPtr(OwnPtr const&) = delete; template NonnullLockRefPtr& operator=(OwnPtr const&) = delete; template NonnullLockRefPtr(LockRefPtr const&) = delete; template NonnullLockRefPtr& operator=(LockRefPtr const&) = delete; NonnullLockRefPtr(LockRefPtr const&) = delete; NonnullLockRefPtr& operator=(LockRefPtr const&) = delete; NonnullLockRefPtr& operator=(NonnullLockRefPtr const& other) { if (this != &other) assign(other.add_ref()); return *this; } template NonnullLockRefPtr& operator=(NonnullLockRefPtr const& other) requires(IsConvertible) { assign(other.add_ref()); return *this; } ALWAYS_INLINE NonnullLockRefPtr& operator=(NonnullLockRefPtr&& other) { if (this != &other) assign(&other.leak_ref()); return *this; } template NonnullLockRefPtr& operator=(NonnullLockRefPtr&& other) requires(IsConvertible) { assign(&other.leak_ref()); return *this; } NonnullLockRefPtr& operator=(T const& object) { const_cast(object).ref(); assign(const_cast(&object)); return *this; } [[nodiscard]] ALWAYS_INLINE T& leak_ref() { T* ptr = exchange(nullptr); VERIFY(ptr); return *ptr; } ALWAYS_INLINE RETURNS_NONNULL T* ptr() { return as_nonnull_ptr(); } ALWAYS_INLINE RETURNS_NONNULL T const* ptr() const { return as_nonnull_ptr(); } ALWAYS_INLINE RETURNS_NONNULL T* operator->() { return as_nonnull_ptr(); } ALWAYS_INLINE RETURNS_NONNULL T const* operator->() const { return as_nonnull_ptr(); } ALWAYS_INLINE T& operator*() { return *as_nonnull_ptr(); } ALWAYS_INLINE T const& operator*() const { return *as_nonnull_ptr(); } ALWAYS_INLINE RETURNS_NONNULL operator T*() { return as_nonnull_ptr(); } ALWAYS_INLINE RETURNS_NONNULL operator T const*() const { return as_nonnull_ptr(); } ALWAYS_INLINE operator T&() { return *as_nonnull_ptr(); } ALWAYS_INLINE operator T const&() const { return *as_nonnull_ptr(); } operator bool() const = delete; bool operator!() const = delete; void swap(NonnullLockRefPtr& other) { if (this == &other) return; // NOTE: swap is not atomic! T* other_ptr = other.exchange(nullptr); T* ptr = exchange(other_ptr); other.exchange(ptr); } template void swap(NonnullLockRefPtr& other) requires(IsConvertible) { // NOTE: swap is not atomic! U* other_ptr = other.exchange(nullptr); T* ptr = exchange(other_ptr); other.exchange(ptr); } // clang-format off private: NonnullLockRefPtr() = delete; // clang-format on ALWAYS_INLINE T* as_ptr() const { return (T*)(m_bits.load(AK::MemoryOrder::memory_order_relaxed) & ~(FlatPtr)1); } ALWAYS_INLINE RETURNS_NONNULL T* as_nonnull_ptr() const { T* ptr = (T*)(m_bits.load(AK::MemoryOrder::memory_order_relaxed) & ~(FlatPtr)1); VERIFY(ptr); return ptr; } template void do_while_locked(F f) const { #ifdef KERNEL // We don't want to be pre-empted while we have the lock bit set Kernel::ScopedCritical critical; #endif FlatPtr bits; for (;;) { bits = m_bits.fetch_or(1, AK::MemoryOrder::memory_order_acq_rel); if (!(bits & 1)) break; #ifdef KERNEL Kernel::Processor::wait_check(); #endif } VERIFY(!(bits & 1)); f((T*)bits); m_bits.store(bits, AK::MemoryOrder::memory_order_release); } ALWAYS_INLINE void assign(T* new_ptr) { T* prev_ptr = exchange(new_ptr); unref_if_not_null(prev_ptr); } ALWAYS_INLINE T* exchange(T* new_ptr) { VERIFY(!((FlatPtr)new_ptr & 1)); #ifdef KERNEL // We don't want to be pre-empted while we have the lock bit set Kernel::ScopedCritical critical; #endif // Only exchange while not locked FlatPtr expected = m_bits.load(AK::MemoryOrder::memory_order_relaxed); for (;;) { expected &= ~(FlatPtr)1; // only if lock bit is not set if (m_bits.compare_exchange_strong(expected, (FlatPtr)new_ptr, AK::MemoryOrder::memory_order_acq_rel)) break; #ifdef KERNEL Kernel::Processor::wait_check(); #endif } VERIFY(!(expected & 1)); return (T*)expected; } T* add_ref() const { #ifdef KERNEL // We don't want to be pre-empted while we have the lock bit set Kernel::ScopedCritical critical; #endif // Lock the pointer FlatPtr expected = m_bits.load(AK::MemoryOrder::memory_order_relaxed); for (;;) { expected &= ~(FlatPtr)1; // only if lock bit is not set if (m_bits.compare_exchange_strong(expected, expected | 1, AK::MemoryOrder::memory_order_acq_rel)) break; #ifdef KERNEL Kernel::Processor::wait_check(); #endif } // Add a reference now that we locked the pointer ref_if_not_null((T*)expected); // Unlock the pointer again m_bits.store(expected, AK::MemoryOrder::memory_order_release); return (T*)expected; } mutable Atomic m_bits { 0 }; }; template inline NonnullLockRefPtr adopt_lock_ref(T& object) { return NonnullLockRefPtr(NonnullLockRefPtr::Adopt, object); } template struct Formatter> : Formatter { ErrorOr format(FormatBuilder& builder, NonnullLockRefPtr const& value) { return Formatter::format(builder, value.ptr()); } }; template inline void swap(NonnullLockRefPtr& a, NonnullLockRefPtr& b) requires(IsConvertible) { a.swap(b); } } template struct Traits> : public GenericTraits> { using PeekType = T*; using ConstPeekType = T const*; static unsigned hash(NonnullLockRefPtr const& p) { return ptr_hash(p.ptr()); } static bool equals(NonnullLockRefPtr const& a, NonnullLockRefPtr const& b) { return a.ptr() == b.ptr(); } }; using AK::adopt_lock_ref; using AK::NonnullLockRefPtr;