diff options
author | Andreas Kling <kling@serenityos.org> | 2021-07-25 01:46:44 +0200 |
---|---|---|
committer | Andreas Kling <kling@serenityos.org> | 2021-07-25 17:28:05 +0200 |
commit | 2d1a651e0aa8be1836a07af2bbae5dfbe522de09 (patch) | |
tree | deb03d26c5278c4fde16820a0f30dba23c9d868f | |
parent | 6bb53d6a80c9bc7e88876fa6774a253aa0ec2af5 (diff) | |
download | serenity-2d1a651e0aa8be1836a07af2bbae5dfbe522de09.zip |
Kernel: Make purgeable memory a VMObject level concept (again)
This patch changes the semantics of purgeable memory.
- AnonymousVMObject now has a "purgeable" flag. It can only be set when
constructing the object. (Previously, all anonymous memory was
effectively purgeable.)
- AnonymousVMObject now has a "volatile" flag. It covers the entire
range of physical pages. (Previously, we tracked ranges of volatile
pages, effectively making it a page-level concept.)
- Non-volatile objects maintain a physical page reservation via the
committed pages mechanism, to ensure full coverage for page faults.
- When an object is made volatile, it relinquishes any unused committed
pages immediately. If later made non-volatile again, we then attempt
to make a new committed pages reservation. If this fails, we return
ENOMEM to userspace.
mmap() now creates purgeable objects if passed the MAP_PURGEABLE option
together with MAP_ANONYMOUS. anon_create() memory is always purgeable.
-rw-r--r-- | Kernel/CMakeLists.txt | 1 | ||||
-rw-r--r-- | Kernel/FileSystem/AnonymousFile.h | 1 | ||||
-rw-r--r-- | Kernel/ProcessSpecificExposed.cpp | 2 | ||||
-rw-r--r-- | Kernel/Syscalls/anon_create.cpp | 2 | ||||
-rw-r--r-- | Kernel/Syscalls/mmap.cpp | 29 | ||||
-rw-r--r-- | Kernel/Syscalls/purge.cpp | 2 | ||||
-rw-r--r-- | Kernel/UnixTypes.h | 1 | ||||
-rw-r--r-- | Kernel/VM/AnonymousVMObject.cpp | 324 | ||||
-rw-r--r-- | Kernel/VM/AnonymousVMObject.h | 125 | ||||
-rw-r--r-- | Kernel/VM/PurgeablePageRanges.cpp | 310 | ||||
-rw-r--r-- | Kernel/VM/PurgeablePageRanges.h | 245 | ||||
-rw-r--r-- | Kernel/VM/Region.cpp | 112 | ||||
-rw-r--r-- | Kernel/VM/Region.h | 17 | ||||
-rw-r--r-- | Kernel/VM/Space.cpp | 10 | ||||
-rw-r--r-- | Userland/Libraries/LibC/malloc.cpp | 2 | ||||
-rw-r--r-- | Userland/Libraries/LibC/sys/mman.h | 1 | ||||
-rw-r--r-- | Userland/Libraries/LibGfx/Bitmap.cpp | 5 |
17 files changed, 187 insertions, 1002 deletions
diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index 686b676d33..2efc36336a 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -264,7 +264,6 @@ set(KERNEL_SOURCES VM/PhysicalZone.cpp VM/PrivateInodeVMObject.cpp VM/ProcessPagingScope.cpp - VM/PurgeablePageRanges.cpp VM/Range.cpp VM/RangeAllocator.cpp VM/Region.cpp diff --git a/Kernel/FileSystem/AnonymousFile.h b/Kernel/FileSystem/AnonymousFile.h index ac0fa0c5e5..1008389754 100644 --- a/Kernel/FileSystem/AnonymousFile.h +++ b/Kernel/FileSystem/AnonymousFile.h @@ -7,6 +7,7 @@ #pragma once #include <Kernel/FileSystem/File.h> +#include <Kernel/VM/AnonymousVMObject.h> namespace Kernel { diff --git a/Kernel/ProcessSpecificExposed.cpp b/Kernel/ProcessSpecificExposed.cpp index f585809109..5b2feebf63 100644 --- a/Kernel/ProcessSpecificExposed.cpp +++ b/Kernel/ProcessSpecificExposed.cpp @@ -456,7 +456,7 @@ private: region_object.add("syscall", region->is_syscall_region()); region_object.add("purgeable", region->vmobject().is_anonymous()); if (region->vmobject().is_anonymous()) { - region_object.add("volatile", static_cast<const AnonymousVMObject&>(region->vmobject()).is_any_volatile()); + region_object.add("volatile", static_cast<AnonymousVMObject const&>(region->vmobject()).is_volatile()); } region_object.add("cacheable", region->is_cacheable()); region_object.add("address", region->vaddr().get()); diff --git a/Kernel/Syscalls/anon_create.cpp b/Kernel/Syscalls/anon_create.cpp index 6a83b9f0d7..35344af9ea 100644 --- a/Kernel/Syscalls/anon_create.cpp +++ b/Kernel/Syscalls/anon_create.cpp @@ -29,7 +29,7 @@ KResultOr<FlatPtr> Process::sys$anon_create(size_t size, int options) if (new_fd < 0) return new_fd; - auto vmobject = AnonymousVMObject::try_create_with_size(size, AllocationStrategy::Reserve); + auto vmobject = AnonymousVMObject::try_create_purgeable_with_size(size, AllocationStrategy::Reserve); if (!vmobject) return ENOMEM; diff --git a/Kernel/Syscalls/mmap.cpp b/Kernel/Syscalls/mmap.cpp index 0ff97eb6e4..eb89039a3a 100644 --- a/Kernel/Syscalls/mmap.cpp +++ b/Kernel/Syscalls/mmap.cpp @@ -12,6 +12,7 @@ #include <Kernel/PerformanceEventBuffer.h> #include <Kernel/PerformanceManager.h> #include <Kernel/Process.h> +#include <Kernel/VM/AnonymousVMObject.h> #include <Kernel/VM/MemoryManager.h> #include <Kernel/VM/PageDirectory.h> #include <Kernel/VM/PrivateInodeVMObject.h> @@ -217,7 +218,14 @@ KResultOr<FlatPtr> Process::sys$mmap(Userspace<const Syscall::SC_mmap_params*> u if (map_anonymous) { auto strategy = map_noreserve ? AllocationStrategy::None : AllocationStrategy::Reserve; - auto region_or_error = space().allocate_region(range.value(), {}, prot, strategy); + RefPtr<AnonymousVMObject> vmobject; + if (flags & MAP_PURGEABLE) + vmobject = AnonymousVMObject::try_create_purgeable_with_size(page_round_up(size), strategy); + else + vmobject = AnonymousVMObject::try_create_with_size(page_round_up(size), strategy); + if (!vmobject) + return ENOMEM; + auto region_or_error = space().allocate_region_with_vmobject(range.value(), vmobject.release_nonnull(), 0, {}, prot, map_shared); if (region_or_error.is_error()) return region_or_error.error().error(); region = region_or_error.value(); @@ -465,23 +473,17 @@ KResultOr<FlatPtr> Process::sys$madvise(Userspace<void*> address, size_t size, i if (set_volatile || set_nonvolatile) { if (!region->vmobject().is_anonymous()) return EPERM; + auto& vmobject = static_cast<AnonymousVMObject&>(region->vmobject()); bool was_purged = false; - switch (region->set_volatile(VirtualAddress(address), size, set_volatile, was_purged)) { - case Region::SetVolatileError::Success: - break; - case Region::SetVolatileError::NotPurgeable: - return EPERM; - case Region::SetVolatileError::OutOfMemory: - return ENOMEM; - } - if (set_nonvolatile) - return was_purged ? 1 : 0; - return 0; + auto result = vmobject.set_volatile(set_volatile, was_purged); + if (result.is_error()) + return result.error(); + return was_purged ? 1 : 0; } if (advice & MADV_GET_VOLATILE) { if (!region->vmobject().is_anonymous()) return EPERM; - return region->is_volatile(VirtualAddress(address), size) ? 0 : 1; + return static_cast<AnonymousVMObject&>(region->vmobject()).is_volatile() ? 0 : 1; } return EINVAL; } @@ -668,5 +670,4 @@ KResultOr<FlatPtr> Process::sys$msyscall(Userspace<void*> address) region->set_syscall_region(true); return 0; } - } diff --git a/Kernel/Syscalls/purge.cpp b/Kernel/Syscalls/purge.cpp index f12bbe6dda..1a58486bc1 100644 --- a/Kernel/Syscalls/purge.cpp +++ b/Kernel/Syscalls/purge.cpp @@ -19,7 +19,7 @@ KResultOr<FlatPtr> Process::sys$purge(int mode) REQUIRE_NO_PROMISES; if (!is_superuser()) return EPERM; - int purged_page_count = 0; + size_t purged_page_count = 0; if (mode & PURGE_ALL_VOLATILE) { NonnullRefPtrVector<AnonymousVMObject> vmobjects; { diff --git a/Kernel/UnixTypes.h b/Kernel/UnixTypes.h index 4ca23090fe..58134b3b53 100644 --- a/Kernel/UnixTypes.h +++ b/Kernel/UnixTypes.h @@ -92,6 +92,7 @@ enum { #define MAP_STACK 0x40 #define MAP_NORESERVE 0x80 #define MAP_RANDOMIZED 0x100 +#define MAP_PURGEABLE 0x200 #define PROT_READ 0x1 #define PROT_WRITE 0x2 diff --git a/Kernel/VM/AnonymousVMObject.cpp b/Kernel/VM/AnonymousVMObject.cpp index 918f9ca099..a65e25a4a6 100644 --- a/Kernel/VM/AnonymousVMObject.cpp +++ b/Kernel/VM/AnonymousVMObject.cpp @@ -22,16 +22,17 @@ RefPtr<VMObject> AnonymousVMObject::try_clone() // commit the number of pages that we need to potentially allocate // so that the parent is still guaranteed to be able to have all // non-volatile memory available. - size_t need_cow_pages = 0; + size_t new_cow_pages_needed = 0; - // We definitely need to commit non-volatile areas - for_each_nonvolatile_range([&](VolatilePageRange const& nonvolatile_range) { - need_cow_pages += nonvolatile_range.count; - }); + if (is_volatile()) { + // NOTE: If this object is currently volatile, we don't own any committed pages. + } else { + new_cow_pages_needed = page_count(); + } - dbgln_if(COMMIT_DEBUG, "Cloning {:p}, need {} committed cow pages", this, need_cow_pages); + dbgln_if(COMMIT_DEBUG, "Cloning {:p}, need {} committed cow pages", this, new_cow_pages_needed); - if (!MM.commit_user_physical_pages(need_cow_pages)) + if (!MM.commit_user_physical_pages(new_cow_pages_needed)) return {}; // Create or replace the committed cow pages. When cloning a previously @@ -40,10 +41,10 @@ RefPtr<VMObject> AnonymousVMObject::try_clone() // one would keep the one it still has. This ensures that the original // one and this one, as well as the clone have sufficient resources // to cow all pages as needed - m_shared_committed_cow_pages = try_create<CommittedCowPages>(need_cow_pages); + m_shared_committed_cow_pages = try_create<CommittedCowPages>(new_cow_pages_needed); if (!m_shared_committed_cow_pages) { - MM.uncommit_user_physical_pages(need_cow_pages); + MM.uncommit_user_physical_pages(new_cow_pages_needed); return {}; } @@ -65,6 +66,20 @@ RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_size(size_t size, A return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(size, commit)); } +RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_purgeable_with_size(size_t size, AllocationStrategy commit) +{ + if (commit == AllocationStrategy::Reserve || commit == AllocationStrategy::AllocateNow) { + // We need to attempt to commit before actually creating the object + if (!MM.commit_user_physical_pages(ceil_div(size, static_cast<size_t>(PAGE_SIZE)))) + return {}; + } + auto vmobject = adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(size, commit)); + if (!vmobject) + return {}; + vmobject->m_purgeable = true; + return vmobject; +} + RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_physical_pages(Span<NonnullRefPtr<PhysicalPage>> physical_pages) { return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(physical_pages)); @@ -81,7 +96,6 @@ RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_for_physical_range(Physi AnonymousVMObject::AnonymousVMObject(size_t size, AllocationStrategy strategy) : VMObject(size) - , m_volatile_ranges_cache({ 0, page_count() }) , m_unused_committed_pages(strategy == AllocationStrategy::Reserve ? page_count() : 0) { if (strategy == AllocationStrategy::AllocateNow) { @@ -97,7 +111,6 @@ AnonymousVMObject::AnonymousVMObject(size_t size, AllocationStrategy strategy) AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size) : VMObject(size) - , m_volatile_ranges_cache({ 0, page_count() }) { VERIFY(paddr.page_base() == paddr); for (size_t i = 0; i < page_count(); ++i) @@ -106,7 +119,6 @@ AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size) AnonymousVMObject::AnonymousVMObject(Span<NonnullRefPtr<PhysicalPage>> physical_pages) : VMObject(physical_pages.size() * PAGE_SIZE) - , m_volatile_ranges_cache({ 0, page_count() }) { for (size_t i = 0; i < physical_pages.size(); ++i) { m_physical_pages[i] = physical_pages[i]; @@ -115,9 +127,6 @@ AnonymousVMObject::AnonymousVMObject(Span<NonnullRefPtr<PhysicalPage>> physical_ AnonymousVMObject::AnonymousVMObject(AnonymousVMObject const& other) : VMObject(other) - , m_volatile_ranges_cache({ 0, page_count() }) // do *not* clone this - , m_volatile_ranges_cache_dirty(true) // do *not* clone this - , m_purgeable_ranges() // do *not* clone this , m_unused_committed_pages(other.m_unused_committed_pages) , m_cow_map() // do *not* clone this , m_shared_committed_cow_pages(other.m_shared_committed_cow_pages) // share the pool @@ -152,217 +161,94 @@ AnonymousVMObject::~AnonymousVMObject() MM.uncommit_user_physical_pages(m_unused_committed_pages); } -int AnonymousVMObject::purge() +size_t AnonymousVMObject::purge() { - int purged_page_count = 0; ScopedSpinLock lock(m_lock); - for_each_volatile_range([&](auto const& range) { - int purged_in_range = 0; - auto range_end = range.base + range.count; - for (size_t i = range.base; i < range_end; i++) { - auto& phys_page = m_physical_pages[i]; - if (phys_page && !phys_page->is_shared_zero_page()) { - VERIFY(!phys_page->is_lazy_committed_page()); - ++purged_in_range; - } - phys_page = MM.shared_zero_page(); - } - - if (purged_in_range > 0) { - purged_page_count += purged_in_range; - set_was_purged(range); - for_each_region([&](auto& region) { - if (auto owner = region.get_owner()) { - // we need to hold a reference the process here (if there is one) as we may not own this region - dmesgln("Purged {} pages from region {} owned by {} at {} - {}", - purged_in_range, - region.name(), - *owner, - region.vaddr_from_page_index(range.base), - region.vaddr_from_page_index(range.base + range.count)); - } else { - dmesgln("Purged {} pages from region {} (no ownership) at {} - {}", - purged_in_range, - region.name(), - region.vaddr_from_page_index(range.base), - region.vaddr_from_page_index(range.base + range.count)); - } - region.remap_vmobject_page_range(range.base, range.count); - }); - } - }); - return purged_page_count; -} -void AnonymousVMObject::set_was_purged(VolatilePageRange const& range) -{ - VERIFY(m_lock.is_locked()); - for (auto* purgeable_ranges : m_purgeable_ranges) - purgeable_ranges->set_was_purged(range); -} + if (!is_purgeable() || !is_volatile()) + return 0; -void AnonymousVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges) -{ - ScopedSpinLock lock(m_lock); - purgeable_page_ranges.set_vmobject(this); - VERIFY(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges)); - m_purgeable_ranges.append(&purgeable_page_ranges); -} + size_t total_pages_purged = 0; -void AnonymousVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges) -{ - ScopedSpinLock lock(m_lock); - for (size_t i = 0; i < m_purgeable_ranges.size(); i++) { - if (m_purgeable_ranges[i] != &purgeable_page_ranges) + for (auto& page : m_physical_pages) { + VERIFY(page); + if (page->is_shared_zero_page()) continue; - purgeable_page_ranges.set_vmobject(nullptr); - m_purgeable_ranges.remove(i); - return; + page = MM.shared_zero_page(); + ++total_pages_purged; } - VERIFY_NOT_REACHED(); -} -bool AnonymousVMObject::is_any_volatile() const -{ - ScopedSpinLock lock(m_lock); - for (auto& volatile_ranges : m_purgeable_ranges) { - ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock); - if (!volatile_ranges->is_empty()) - return true; - } - return false; -} + m_was_purged = true; -size_t AnonymousVMObject::remove_lazy_commit_pages(VolatilePageRange const& range) -{ - VERIFY(m_lock.is_locked()); - - size_t removed_count = 0; - auto range_end = range.base + range.count; - for (size_t i = range.base; i < range_end; i++) { - auto& phys_page = m_physical_pages[i]; - if (phys_page && phys_page->is_lazy_committed_page()) { - phys_page = MM.shared_zero_page(); - removed_count++; - VERIFY(m_unused_committed_pages > 0); - if (--m_unused_committed_pages == 0) - break; - } - } - return removed_count; + for_each_region([](Region& region) { + region.remap(); + }); + + return total_pages_purged; } -void AnonymousVMObject::update_volatile_cache() +KResult AnonymousVMObject::set_volatile(bool is_volatile, bool& was_purged) { - VERIFY(m_lock.is_locked()); - VERIFY(m_volatile_ranges_cache_dirty); + VERIFY(is_purgeable()); - m_volatile_ranges_cache.clear(); - for_each_nonvolatile_range([&](VolatilePageRange const& range) { - m_volatile_ranges_cache.add_unchecked(range); - }); + ScopedSpinLock locker(m_lock); - m_volatile_ranges_cache_dirty = false; -} + was_purged = m_was_purged; + if (m_volatile == is_volatile) + return KSuccess; -void AnonymousVMObject::range_made_volatile(VolatilePageRange const& range) -{ - VERIFY(m_lock.is_locked()); - - if (m_unused_committed_pages == 0) - return; - - // We need to check this range for any pages that are marked for - // lazy committed allocation and turn them into shared zero pages - // and also adjust the m_unused_committed_pages for each such page. - // Take into account all the other views as well. - size_t uncommit_page_count = 0; - for_each_volatile_range([&](auto const& r) { - auto intersected = range.intersected(r); - if (!intersected.is_empty()) { - uncommit_page_count += remove_lazy_commit_pages(intersected); - if (m_unused_committed_pages == 0) - return IterationDecision::Break; + if (is_volatile) { + // When a VMObject is made volatile, it gives up all of its committed memory. + // Any physical pages already allocated remain in the VMObject for now, but the kernel is free to take them at any moment. + for (auto& page : m_physical_pages) { + if (page && page->is_lazy_committed_page()) + page = MM.shared_zero_page(); } - return IterationDecision::Continue; - }); - // Return those committed pages back to the system - if (uncommit_page_count > 0) { - dbgln_if(COMMIT_DEBUG, "Uncommit {} lazy-commit pages from {:p}", uncommit_page_count, this); - MM.uncommit_user_physical_pages(uncommit_page_count); + if (m_unused_committed_pages) { + MM.uncommit_user_physical_pages(m_unused_committed_pages); + m_unused_committed_pages = 0; + } + + m_volatile = true; + m_was_purged = false; + return KSuccess; + } + // When a VMObject is made non-volatile, we try to commit however many pages are not currently available. + // If that fails, we return false to indicate that memory allocation failed. + size_t committed_pages_needed = 0; + for (auto& page : m_physical_pages) { + VERIFY(page); + if (page->is_shared_zero_page()) + ++committed_pages_needed; } - m_volatile_ranges_cache_dirty = true; -} + if (!committed_pages_needed) { + m_volatile = false; + return KSuccess; + } -void AnonymousVMObject::range_made_nonvolatile(VolatilePageRange const&) -{ - VERIFY(m_lock.is_locked()); - m_volatile_ranges_cache_dirty = true; -} + if (!MM.commit_user_physical_pages(committed_pages_needed)) + return ENOMEM; -size_t AnonymousVMObject::count_needed_commit_pages_for_nonvolatile_range(VolatilePageRange const& range) -{ - VERIFY(m_lock.is_locked()); - VERIFY(!range.is_empty()); - - size_t need_commit_pages = 0; - auto range_end = range.base + range.count; - for (size_t page_index = range.base; page_index < range_end; page_index++) { - // COW pages are accounted for in m_shared_committed_cow_pages - if (!m_cow_map.is_null() && m_cow_map.get(page_index)) - continue; - auto& phys_page = m_physical_pages[page_index]; - if (phys_page && phys_page->is_shared_zero_page()) - need_commit_pages++; - } - return need_commit_pages; -} + m_unused_committed_pages = committed_pages_needed; -size_t AnonymousVMObject::mark_committed_pages_for_nonvolatile_range(VolatilePageRange const& range, size_t mark_total) -{ - VERIFY(m_lock.is_locked()); - VERIFY(!range.is_empty()); - VERIFY(mark_total > 0); - - size_t pages_updated = 0; - auto range_end = range.base + range.count; - for (size_t page_index = range.base; page_index < range_end; page_index++) { - // COW pages are accounted for in m_shared_committed_cow_pages - if (!m_cow_map.is_null() && m_cow_map.get(page_index)) - continue; - auto& phys_page = m_physical_pages[page_index]; - if (phys_page && phys_page->is_shared_zero_page()) { - phys_page = MM.lazy_committed_page(); - if (++pages_updated == mark_total) - break; - } + for (auto& page : m_physical_pages) { + if (page->is_shared_zero_page()) + page = MM.lazy_committed_page(); } - dbgln_if(COMMIT_DEBUG, "Added {} lazy-commit pages to {:p}", pages_updated, this); - - m_unused_committed_pages += pages_updated; - return pages_updated; + m_volatile = false; + m_was_purged = false; + return KSuccess; } -NonnullRefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(Badge<Region>, size_t page_index) +NonnullRefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(Badge<Region>) { { ScopedSpinLock lock(m_lock); - VERIFY(m_unused_committed_pages > 0); - - // We shouldn't have any committed page tags in volatile regions - VERIFY([&]() { - for (auto* purgeable_ranges : m_purgeable_ranges) { - if (purgeable_ranges->is_volatile(page_index)) - return false; - } - return true; - }()); - - m_unused_committed_pages--; + --m_unused_committed_pages; } return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes); } @@ -404,19 +290,20 @@ size_t AnonymousVMObject::cow_pages() const return m_cow_map.count_slow(true); } -bool AnonymousVMObject::is_nonvolatile(size_t page_index) -{ - if (m_volatile_ranges_cache_dirty) - update_volatile_cache(); - return !m_volatile_ranges_cache.contains(page_index); -} - PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, VirtualAddress vaddr) { VERIFY_INTERRUPTS_DISABLED(); ScopedSpinLock lock(m_lock); + + if (is_volatile()) { + // A COW fault in a volatile region? Userspace is writing to volatile memory, this is a bug. Crash. + dbgln("COW fault in volatile region, will crash."); + return PageFaultResponse::ShouldCrash; + } + auto& page_slot = physical_pages()[page_index]; - bool have_committed = m_shared_committed_cow_pages && is_nonvolatile(page_index); + bool have_committed = m_shared_committed_cow_pages; + if (page_slot->ref_count() == 1) { dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page but nobody is sharing it anymore. Remap r/w"); set_should_cow(page_index, false); @@ -462,4 +349,33 @@ PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, Virtual return PageFaultResponse::Continue; } +CommittedCowPages::CommittedCowPages(size_t committed_pages) + : m_committed_pages(committed_pages) +{ +} + +CommittedCowPages::~CommittedCowPages() +{ + // Return unused committed pages + if (m_committed_pages > 0) + MM.uncommit_user_physical_pages(m_committed_pages); +} + +NonnullRefPtr<PhysicalPage> CommittedCowPages::allocate_one() +{ + VERIFY(m_committed_pages > 0); + m_committed_pages--; + + return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes); +} + +bool CommittedCowPages::return_one() +{ + VERIFY(m_committed_pages > 0); + m_committed_pages--; + + MM.uncommit_user_physical_pages(1); + return m_committed_pages == 0; +} + } diff --git a/Kernel/VM/AnonymousVMObject.h b/Kernel/VM/AnonymousVMObject.h index 1b1a721695..f238c52e2d 100644 --- a/Kernel/VM/AnonymousVMObject.h +++ b/Kernel/VM/AnonymousVMObject.h @@ -10,108 +10,48 @@ #include <Kernel/VM/AllocationStrategy.h> #include <Kernel/VM/MemoryManager.h> #include <Kernel/VM/PageFaultResponse.h> -#include <Kernel/VM/PurgeablePageRanges.h> #include <Kernel/VM/VMObject.h> namespace Kernel { -class AnonymousVMObject final : public VMObject { - friend class PurgeablePageRanges; +class CommittedCowPages : public RefCounted<CommittedCowPages> { + AK_MAKE_NONCOPYABLE(CommittedCowPages); + +public: + CommittedCowPages() = delete; + + CommittedCowPages(size_t); + ~CommittedCowPages(); + + NonnullRefPtr<PhysicalPage> allocate_one(); + bool return_one(); + +private: + size_t m_committed_pages; +}; +class AnonymousVMObject final : public VMObject { public: virtual ~AnonymousVMObject() override; static RefPtr<AnonymousVMObject> try_create_with_size(size_t, AllocationStrategy); static RefPtr<AnonymousVMObject> try_create_for_physical_range(PhysicalAddress paddr, size_t size); static RefPtr<AnonymousVMObject> try_create_with_physical_pages(Span<NonnullRefPtr<PhysicalPage>>); + static RefPtr<AnonymousVMObject> try_create_purgeable_with_size(size_t, AllocationStrategy); virtual RefPtr<VMObject> try_clone() override; - [[nodiscard]] NonnullRefPtr<PhysicalPage> allocate_committed_page(Badge<Region>, size_t); + [[nodiscard]] NonnullRefPtr<PhysicalPage> allocate_committed_page(Badge<Region>); PageFaultResponse handle_cow_fault(size_t, VirtualAddress); size_t cow_pages() const; bool should_cow(size_t page_index, bool) const; void set_should_cow(size_t page_index, bool); - void register_purgeable_page_ranges(PurgeablePageRanges&); - void unregister_purgeable_page_ranges(PurgeablePageRanges&); - - int purge(); - - bool is_any_volatile() const; - - template<IteratorFunction<VolatilePageRange const&> F> - IterationDecision for_each_volatile_range(F f) const - { - VERIFY(m_lock.is_locked()); - // This is a little ugly. Basically, we're trying to find the - // volatile ranges that all share, because those are the only - // pages we can actually purge - for (auto* purgeable_range : m_purgeable_ranges) { - ScopedSpinLock purgeable_lock(purgeable_range->m_volatile_ranges_lock); - for (auto& r1 : purgeable_range->volatile_ranges().ranges()) { - VolatilePageRange range(r1); - for (auto* purgeable_range2 : m_purgeable_ranges) { - if (purgeable_range2 == purgeable_range) - continue; - ScopedSpinLock purgeable2_lock(purgeable_range2->m_volatile_ranges_lock); - if (purgeable_range2->is_empty()) { - // If just one doesn't allow any purging, we can - // immediately bail - return IterationDecision::Continue; - } - for (auto const& r2 : purgeable_range2->volatile_ranges().ranges()) { - range = range.intersected(r2); - if (range.is_empty()) - break; - } - if (range.is_empty()) - break; - } - if (range.is_empty()) - continue; - IterationDecision decision = f(range); - if (decision != IterationDecision::Continue) - return decision; - } - } - return IterationDecision::Continue; - } - - template<IteratorFunction<VolatilePageRange const&> F> - IterationDecision for_each_nonvolatile_range(F f) const - { - size_t base = 0; - for_each_volatile_range([&](VolatilePageRange const& volatile_range) { - if (volatile_range.base == base) - return IterationDecision::Continue; - IterationDecision decision = f(VolatilePageRange { base, volatile_range.base - base }); - if (decision != IterationDecision::Continue) - return decision; - base = volatile_range.base + volatile_range.count; - return IterationDecision::Continue; - }); - if (base < page_count()) - return f(VolatilePageRange { base, page_count() - base }); - return IterationDecision::Continue; - } - - template<VoidFunction<VolatilePageRange const&> F> - IterationDecision for_each_volatile_range(F f) const - { - return for_each_volatile_range([&](auto& range) { - f(range); - return IterationDecision::Continue; - }); - } - - template<VoidFunction<VolatilePageRange const&> F> - IterationDecision for_each_nonvolatile_range(F f) const - { - return for_each_nonvolatile_range([&](auto range) { - f(move(range)); - return IterationDecision::Continue; - }); - } + bool is_purgeable() const { return m_purgeable; } + bool is_volatile() const { return m_volatile; } + + KResult set_volatile(bool is_volatile, bool& was_purged); + + size_t purge(); private: explicit AnonymousVMObject(size_t, AllocationStrategy); @@ -121,15 +61,6 @@ private: virtual StringView class_name() const override { return "AnonymousVMObject"sv; } - void update_volatile_cache(); - void set_was_purged(VolatilePageRange const&); - size_t remove_lazy_commit_pages(VolatilePageRange const&); - void range_made_volatile(VolatilePageRange const&); - void range_made_nonvolatile(VolatilePageRange const&); - size_t count_needed_commit_pages_for_nonvolatile_range(VolatilePageRange const&); - size_t mark_committed_pages_for_nonvolatile_range(VolatilePageRange const&, size_t); - bool is_nonvolatile(size_t page_index); - AnonymousVMObject& operator=(AnonymousVMObject const&) = delete; AnonymousVMObject& operator=(AnonymousVMObject&&) = delete; AnonymousVMObject(AnonymousVMObject&&) = delete; @@ -139,15 +70,15 @@ private: Bitmap& ensure_cow_map(); void ensure_or_reset_cow_map(); - VolatilePageRanges m_volatile_ranges_cache; - bool m_volatile_ranges_cache_dirty { true }; - Vector<PurgeablePageRanges*> m_purgeable_ranges; size_t m_unused_committed_pages { 0 }; - Bitmap m_cow_map; // We share a pool of committed cow-pages with clones RefPtr<CommittedCowPages> m_shared_committed_cow_pages; + + bool m_purgeable { false }; + bool m_volatile { false }; + bool m_was_purged { false }; }; } diff --git a/Kernel/VM/PurgeablePageRanges.cpp b/Kernel/VM/PurgeablePageRanges.cpp deleted file mode 100644 index 0053ad5f5a..0000000000 --- a/Kernel/VM/PurgeablePageRanges.cpp +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include <AK/BinarySearch.h> -#include <AK/ScopeGuard.h> -#include <Kernel/Debug.h> -#include <Kernel/Process.h> -#include <Kernel/VM/AnonymousVMObject.h> -#include <Kernel/VM/MemoryManager.h> -#include <Kernel/VM/PhysicalPage.h> -#include <Kernel/VM/PurgeablePageRanges.h> - -namespace AK { -template<> -struct Formatter<Kernel::VolatilePageRange> : Formatter<String> { - void format(FormatBuilder& builder, const Kernel::VolatilePageRange& value) - { - return Formatter<String>::format(builder, String::formatted("{{{} ({}) purged: {}}}", value.base, value.count, value.was_purged)); - } -}; -} - -namespace Kernel { - -static void dump_volatile_page_ranges(const Vector<VolatilePageRange>& ranges) -{ - if constexpr (VOLATILE_PAGE_RANGES_DEBUG) { - for (size_t i = 0; i < ranges.size(); i++) { - dbgln("[{}] {}", i, ranges[i]); - } - } -} - -void VolatilePageRanges::add_unchecked(const VolatilePageRange& range) -{ - auto add_range = m_total_range.intersected(range); - if (add_range.is_empty()) - return; - m_ranges.append(range); -} - -bool VolatilePageRanges::add(const VolatilePageRange& range) -{ - auto add_range = m_total_range.intersected(range); - if (add_range.is_empty()) - return false; - add_range.was_purged = range.was_purged; - - if constexpr (VOLATILE_PAGE_RANGES_DEBUG) { - dbgln("ADD {} (total range: {}) -->", range, m_total_range); - dump_volatile_page_ranges(m_ranges); - ScopeGuard debug_guard([&]() { - dbgln("After adding {} (total range: {})", range, m_total_range); - dump_volatile_page_ranges(m_ranges); - dbgln("<-- ADD {} (total range: {})", range, m_total_range); - }); - } - - size_t nearby_index = 0; - auto* existing_range = binary_search( - m_ranges.span(), add_range, &nearby_index, [](auto& a, auto& b) { - if (a.intersects_or_adjacent(b)) - return 0; - return (signed)(a.base - (b.base + b.count - 1)); - }); - - size_t inserted_index = 0; - if (existing_range) { - if (*existing_range == add_range) - return false; - - if (existing_range->was_purged != add_range.was_purged) { - // Found an intersecting or adjacent range, but the purge flag - // doesn't match. Subtract what we're adding from it, and - existing_range->subtract_intersecting(add_range); - if (existing_range->is_empty()) { - *existing_range = add_range; - } else { - m_ranges.insert(++nearby_index, add_range); - existing_range = &m_ranges[nearby_index]; - } - } else { - // Found an intersecting or adjacent range that can be merged - existing_range->combine_intersecting_or_adjacent(add_range); - } - inserted_index = nearby_index; - } else { - // Insert into the sorted list - m_ranges.insert_before_matching( - VolatilePageRange(add_range), [&](auto& entry) { - return entry.base >= add_range.base + add_range.count; - }, - nearby_index, &inserted_index); - existing_range = &m_ranges[inserted_index]; - } - - // See if we can merge any of the following ranges - inserted_index++; - while (inserted_index < m_ranges.size()) { - auto& next_range = m_ranges[inserted_index]; - if (!next_range.intersects_or_adjacent(*existing_range)) - break; - if (next_range.was_purged != existing_range->was_purged) { - // The purged flag of following range is not the same. - // Subtract the added/combined range from it - next_range.subtract_intersecting(*existing_range); - if (next_range.is_empty()) - m_ranges.remove(inserted_index); - } else { - existing_range->combine_intersecting_or_adjacent(next_range); - m_ranges.remove(inserted_index); - } - } - return true; -} - -bool VolatilePageRanges::remove(const VolatilePageRange& range, bool& was_purged) -{ - auto remove_range = m_total_range.intersected(range); - if (remove_range.is_empty()) - return false; - - if constexpr (VOLATILE_PAGE_RANGES_DEBUG) { - dbgln("REMOVE {} (total range: {}) -->", range, m_total_range); - dump_volatile_page_ranges(m_ranges); - ScopeGuard debug_guard([&]() { - dbgln("After removing {} (total range: {})", range, m_total_range); - dump_volatile_page_ranges(m_ranges); - dbgln("<-- REMOVE {} (total range: {}) was_purged: {}", range, m_total_range, was_purged); - }); - } - - size_t nearby_index = 0; - auto* existing_range = binary_search( - m_ranges.span(), remove_range, &nearby_index, [](auto& a, auto& b) { - if (a.intersects(b)) - return 0; - return (signed)(a.base - (b.base + b.count - 1)); - }); - if (!existing_range) - return false; - - was_purged = existing_range->was_purged; - if (existing_range->range_equals(remove_range)) { - m_ranges.remove(nearby_index); - } else { - // See if we need to remove any of the following ranges - VERIFY(existing_range == &m_ranges[nearby_index]); // sanity check - while (nearby_index < m_ranges.size()) { - existing_range = &m_ranges[nearby_index]; - if (!existing_range->intersects(range)) - break; - was_purged |= existing_range->was_purged; - existing_range->subtract_intersecting(remove_range); - if (existing_range->is_empty()) { - m_ranges.remove(nearby_index); - break; - } - } - } - return true; -} - -bool VolatilePageRanges::intersects(const VolatilePageRange& range) const -{ - auto* existing_range = binary_search( - m_ranges.span(), range, nullptr, [](auto& a, auto& b) { - if (a.intersects(b)) - return 0; - return (signed)(a.base - (b.base + b.count - 1)); - }); - return existing_range != nullptr; -} - -PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject) - : m_volatile_ranges({ 0, vmobject.is_anonymous() ? vmobject.page_count() : 0 }) -{ -} - -bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range) -{ - if (range.is_empty()) - return false; - - // Since we may need to call into AnonymousVMObject we need to acquire - // its lock as well, and acquire it first. This is important so that - // we don't deadlock when a page fault (e.g. on another processor) - // happens that is meant to lazy-allocate a committed page. It would - // call into AnonymousVMObject::range_made_volatile, which then would - // also call into this object and need to acquire m_lock. By acquiring - // the vmobject lock first in both cases, we avoid deadlocking. - // We can access m_vmobject without any locks for that purpose because - // add_volatile_range and remove_volatile_range can only be called - // by same object that calls set_vmobject. - ScopedSpinLock vmobject_lock(m_vmobject->m_lock); - ScopedSpinLock lock(m_volatile_ranges_lock); - bool added = m_volatile_ranges.add(range); - if (added) - m_vmobject->range_made_volatile(range); - return added; -} - -auto PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged) -> RemoveVolatileError -{ - if (range.is_empty()) { - was_purged = false; - return RemoveVolatileError::Success; - } - ScopedSpinLock vmobject_lock(m_vmobject->m_lock); // see comment in add_volatile_range - ScopedSpinLock lock(m_volatile_ranges_lock); - VERIFY(m_vmobject); - - // Before we actually remove this range, we need to check if we need - // to commit any pages, which may fail. If it fails, we don't actually - // want to make any modifications. COW pages are already accounted for - // in m_shared_committed_cow_pages - size_t need_commit_pages = 0; - m_volatile_ranges.for_each_intersecting_range(range, [&](const VolatilePageRange& intersected_range) { - need_commit_pages += m_vmobject->count_needed_commit_pages_for_nonvolatile_range(intersected_range); - return IterationDecision::Continue; - }); - if (need_commit_pages > 0) { - // See if we can grab enough pages for what we're marking non-volatile - if (!MM.commit_user_physical_pages(need_commit_pages)) - return RemoveVolatileError::OutOfMemory; - - // Now that we are committed to these pages, mark them for lazy-commit allocation - auto pages_to_mark = need_commit_pages; - m_volatile_ranges.for_each_intersecting_range(range, [&](const VolatilePageRange& intersected_range) { - auto pages_marked = m_vmobject->mark_committed_pages_for_nonvolatile_range(intersected_range, pages_to_mark); - pages_to_mark -= pages_marked; - return IterationDecision::Continue; - }); - } - - // Now actually remove the range - if (m_volatile_ranges.remove(range, was_purged)) { - m_vmobject->range_made_nonvolatile(range); - return RemoveVolatileError::Success; - } - - VERIFY(need_commit_pages == 0); // We should have not touched anything - return RemoveVolatileError::SuccessNoChange; -} - -bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) const -{ - if (range.is_empty()) - return false; - ScopedSpinLock lock(m_volatile_ranges_lock); - return m_volatile_ranges.intersects(range); -} - -bool PurgeablePageRanges::is_volatile(size_t index) const -{ - ScopedSpinLock lock(m_volatile_ranges_lock); - return m_volatile_ranges.contains(index); -} - -void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range) -{ - ScopedSpinLock lock(m_volatile_ranges_lock); - m_volatile_ranges.add({ range.base, range.count, true }); -} - -void PurgeablePageRanges::set_vmobject(AnonymousVMObject* vmobject) -{ - // No lock needed here - if (vmobject) { - VERIFY(!m_vmobject); - m_vmobject = vmobject; - } else { - VERIFY(m_vmobject); - m_vmobject = nullptr; - } -} - -CommittedCowPages::CommittedCowPages(size_t committed_pages) - : m_committed_pages(committed_pages) -{ -} - -CommittedCowPages::~CommittedCowPages() -{ - // Return unused committed pages - if (m_committed_pages > 0) - MM.uncommit_user_physical_pages(m_committed_pages); -} - -NonnullRefPtr<PhysicalPage> CommittedCowPages::allocate_one() -{ - VERIFY(m_committed_pages > 0); - m_committed_pages--; - - return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes); -} - -bool CommittedCowPages::return_one() -{ - VERIFY(m_committed_pages > 0); - m_committed_pages--; - - MM.uncommit_user_physical_pages(1); - return m_committed_pages == 0; -} - -} diff --git a/Kernel/VM/PurgeablePageRanges.h b/Kernel/VM/PurgeablePageRanges.h deleted file mode 100644 index 4b7fade270..0000000000 --- a/Kernel/VM/PurgeablePageRanges.h +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org> - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include <AK/Bitmap.h> -#include <AK/RefCounted.h> -#include <Kernel/SpinLock.h> - -namespace Kernel { - -struct VolatilePageRange { - size_t base { 0 }; - size_t count { 0 }; - bool was_purged { false }; - - bool is_empty() const { return count == 0; } - - bool intersects(const VolatilePageRange& other) const - { - return other.base < base + count || other.base + other.count > base; - } - - bool intersects_or_adjacent(const VolatilePageRange& other) const - { - return other.base <= base + count || other.base + other.count >= base; - } - - bool contains(const VolatilePageRange& other) const - { - return base <= other.base && base + count >= other.base + other.count; - } - - VolatilePageRange intersected(const VolatilePageRange& other) const - { - auto b = max(base, other.base); - auto e = min(base + count, other.base + other.count); - if (b >= e) - return {}; - return { b, e - b, was_purged }; - } - - void combine_intersecting_or_adjacent(const VolatilePageRange& other) - { - VERIFY(intersects_or_adjacent(other)); - if (base <= other.base) { - count = (other.base - base) + other.count; - } else { - count = (base - other.base) + count; - base = other.base; - } - was_purged |= other.was_purged; - } - - void subtract_intersecting(const VolatilePageRange& other) - { - if (!intersects(other)) - return; - if (other.contains(*this)) { - count = 0; - return; - } - if (base <= other.base) { - count = (other.base - base); - } else { - auto new_base = other.base + other.count; - count = (base + count) - new_base; - base = new_base; - } - } - - bool range_equals(const VolatilePageRange& other) const - { - return base == other.base && count == other.count; - } - bool operator==(const VolatilePageRange& other) const - { - return base == other.base && count == other.count && was_purged == other.was_purged; - } - bool operator!=(const VolatilePageRange& other) const - { - return base != other.base || count != other.count || was_purged != other.was_purged; - } -}; - -class VolatilePageRanges { -public: - VolatilePageRanges(const VolatilePageRange& total_range) - : m_total_range(total_range) - { - } - VolatilePageRanges(const VolatilePageRanges& other) - : m_ranges(other.m_ranges) - , m_total_range(other.m_total_range) - { - } - - bool is_empty() const { return m_ranges.is_empty(); } - void clear() { m_ranges.clear_with_capacity(); } - - bool is_all() const - { - if (m_ranges.size() != 1) - return false; - return m_ranges[0] == m_total_range; - } - - void set_all() - { - if (m_ranges.size() != 1) - m_ranges = { m_total_range }; - else - m_ranges[0] = m_total_range; - } - - bool intersects(const VolatilePageRange&) const; - bool contains(size_t index) const - { - return intersects({ index, 1 }); - } - - bool add(const VolatilePageRange&); - void add_unchecked(const VolatilePageRange&); - bool remove(const VolatilePageRange&, bool&); - - template<typename F> - IterationDecision for_each_intersecting_range(const VolatilePageRange& range, F f) - { - auto r = m_total_range.intersected(range); - if (r.is_empty()) - return IterationDecision::Continue; - - size_t nearby_index = 0; - auto* existing_range = binary_search( - m_ranges.span(), r, &nearby_index, [](auto& a, auto& b) { - if (a.intersects(b)) - return 0; - return (signed)(a.base - (b.base + b.count - 1)); - }); - if (!existing_range) - return IterationDecision::Continue; - - if (existing_range->range_equals(r)) - return f(r); - VERIFY(existing_range == &m_ranges[nearby_index]); // sanity check - while (nearby_index < m_ranges.size()) { - existing_range = &m_ranges[nearby_index]; - if (!existing_range->intersects(range)) - break; - - IterationDecision decision = f(existing_range->intersected(r)); - if (decision != IterationDecision::Continue) - return decision; - - nearby_index++; - } - return IterationDecision::Continue; - } - - template<typename F> - IterationDecision for_each_nonvolatile_range(F f) const - { - size_t base = m_total_range.base; - for (const auto& volatile_range : m_ranges) { - if (volatile_range.base == base) - continue; - IterationDecision decision = f({ base, volatile_range.base - base }); - if (decision != IterationDecision::Continue) - return decision; - base = volatile_range.base + volatile_range.count; - } - if (base < m_total_range.base + m_total_range.count) - return f({ base, (m_total_range.base + m_total_range.count) - base }); - return IterationDecision::Continue; - } - - Vector<VolatilePageRange>& ranges() { return m_ranges; } - const Vector<VolatilePageRange>& ranges() const { return m_ranges; } - -private: - Vector<VolatilePageRange> m_ranges; - VolatilePageRange m_total_range; -}; - -class AnonymousVMObject; - -class PurgeablePageRanges { - friend class AnonymousVMObject; - -public: - PurgeablePageRanges(const VMObject&); - - void copy_purgeable_page_ranges(const PurgeablePageRanges& other) - { - if (this == &other) - return; - ScopedSpinLock lock(m_volatile_ranges_lock); - ScopedSpinLock other_lock(other.m_volatile_ranges_lock); - m_volatile_ranges = other.m_volatile_ranges; - } - - bool add_volatile_range(const VolatilePageRange& range); - enum class RemoveVolatileError { - Success = 0, - SuccessNoChange, - OutOfMemory - }; - RemoveVolatileError remove_volatile_range(const VolatilePageRange& range, bool& was_purged); - bool is_volatile_range(const VolatilePageRange& range) const; - bool is_volatile(size_t) const; - - bool is_empty() const { return m_volatile_ranges.is_empty(); } - - void set_was_purged(const VolatilePageRange&); - - const VolatilePageRanges& volatile_ranges() const { return m_volatile_ranges; } - -protected: - void set_vmobject(AnonymousVMObject*); - - VolatilePageRanges m_volatile_ranges; - mutable RecursiveSpinLock m_volatile_ranges_lock; - AnonymousVMObject* m_vmobject { nullptr }; -}; - -class CommittedCowPages : public RefCounted<CommittedCowPages> { - AK_MAKE_NONCOPYABLE(CommittedCowPages); - -public: - CommittedCowPages() = delete; - - CommittedCowPages(size_t); - ~CommittedCowPages(); - - NonnullRefPtr<PhysicalPage> allocate_one(); - bool return_one(); - -private: - size_t m_committed_pages; -}; - -} diff --git a/Kernel/VM/Region.cpp b/Kernel/VM/Region.cpp index 5ba828f9b9..6c37f2b029 100644 --- a/Kernel/VM/Region.cpp +++ b/Kernel/VM/Region.cpp @@ -20,8 +20,7 @@ namespace Kernel { Region::Region(Range const& range, NonnullRefPtr<VMObject> vmobject, size_t offset_in_vmobject, OwnPtr<KString> name, Region::Access access, Cacheable cacheable, bool shared) - : PurgeablePageRanges(vmobject) - , m_range(range) + : m_range(range) , m_offset_in_vmobject(offset_in_vmobject) , m_vmobject(move(vmobject)) , m_name(move(name)) @@ -34,14 +33,12 @@ Region::Region(Range const& range, NonnullRefPtr<VMObject> vmobject, size_t offs VERIFY((m_range.size() % PAGE_SIZE) == 0); m_vmobject->add_region(*this); - register_purgeable_page_ranges(); MM.register_region(*this); } Region::~Region() { m_vmobject->remove_region(*this); - unregister_purgeable_page_ranges(); // Make sure we disable interrupts so we don't get interrupted between unmapping and unregistering. // Unmapping the region will give the VM back to the RangeAllocator, so an interrupt handler would @@ -55,22 +52,6 @@ Region::~Region() MM.unregister_region(*this); } -void Region::register_purgeable_page_ranges() -{ - if (m_vmobject->is_anonymous()) { - auto& vmobject = static_cast<AnonymousVMObject&>(*m_vmobject); - vmobject.register_purgeable_page_ranges(*this); - } -} - -void Region::unregister_purgeable_page_ranges() -{ - if (m_vmobject->is_anonymous()) { - auto& vmobject = static_cast<AnonymousVMObject&>(*m_vmobject); - vmobject.unregister_purgeable_page_ranges(*this); - } -} - OwnPtr<Region> Region::clone(Process& new_owner) { VERIFY(Process::current()); @@ -89,8 +70,6 @@ OwnPtr<Region> Region::clone(Process& new_owner) dbgln("Region::clone: Unable to allocate new Region"); return nullptr; } - if (m_vmobject->is_anonymous()) - region->copy_purgeable_page_ranges(*this); region->set_mmap(m_mmap); region->set_shared(m_shared); region->set_syscall_region(is_syscall_region()); @@ -112,8 +91,6 @@ OwnPtr<Region> Region::clone(Process& new_owner) dbgln("Region::clone: Unable to allocate new Region for COW"); return nullptr; } - if (m_vmobject->is_anonymous()) - clone_region->copy_purgeable_page_ranges(*this); if (m_stack) { VERIFY(is_readable()); VERIFY(is_writable()); @@ -129,55 +106,9 @@ void Region::set_vmobject(NonnullRefPtr<VMObject>&& obj) { if (m_vmobject.ptr() == obj.ptr()) return; - unregister_purgeable_page_ranges(); m_vmobject->remove_region(*this); m_vmobject = move(obj); m_vmobject->add_region(*this); - register_purgeable_page_ranges(); -} - -bool Region::is_volatile(VirtualAddress vaddr, size_t size) const -{ - if (!m_vmobject->is_anonymous()) - return false; - - auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject); - size_t first_page_index = page_round_down(offset_in_vmobject) / PAGE_SIZE; - size_t last_page_index = page_round_up(offset_in_vmobject + size) / PAGE_SIZE; - return is_volatile_range({ first_page_index, last_page_index - first_page_index }); -} - -auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged) -> SetVolatileError -{ - was_purged = false; - if (!m_vmobject->is_anonymous()) - return SetVolatileError::NotPurgeable; - - auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject); - if (is_volatile) { - // If marking pages as volatile, be prudent by not marking - // partial pages volatile to prevent potentially non-volatile - // data to be discarded. So rund up the first page and round - // down the last page. - size_t first_page_index = page_round_up(offset_in_vmobject) / PAGE_SIZE; - size_t last_page_index = page_round_down(offset_in_vmobject + size) / PAGE_SIZE; - if (first_page_index != last_page_index) - add_volatile_range({ first_page_index, last_page_index - first_page_index }); - } else { - // If marking pages as non-volatile, round down the first page - // and round up the last page to make sure the beginning and - // end of the range doesn't inadvertedly get discarded. - size_t first_page_index = page_round_down(offset_in_vmobject) / PAGE_SIZE; - size_t last_page_index = page_round_up(offset_in_vmobject + size) / PAGE_SIZE; - switch (remove_volatile_range({ first_page_index, last_page_index - first_page_index }, was_purged)) { - case PurgeablePageRanges::RemoveVolatileError::Success: - case PurgeablePageRanges::RemoveVolatileError::SuccessNoChange: - break; - case PurgeablePageRanges::RemoveVolatileError::OutOfMemory: - return SetVolatileError::OutOfMemory; - } - } - return SetVolatileError::Success; } size_t Region::cow_pages() const @@ -279,43 +210,6 @@ bool Region::map_individual_page_impl(size_t page_index) return true; } -bool Region::do_remap_vmobject_page_range(size_t page_index, size_t page_count) -{ - bool success = true; - if (!m_page_directory) - return success; // not an error, region may have not yet mapped it - if (!translate_vmobject_page_range(page_index, page_count)) - return success; // not an error, region doesn't map this page range - ScopedSpinLock page_lock(m_page_directory->get_lock()); - size_t index = page_index; - while (index < page_index + page_count) { - if (!map_individual_page_impl(index)) { - success = false; - break; - } - index++; - } - if (index > page_index) - MM.flush_tlb(m_page_directory, vaddr_from_page_index(page_index), index - page_index); - return success; -} - -bool Region::remap_vmobject_page_range(size_t page_index, size_t page_count) -{ - bool success = true; - auto& vmobject = this->vmobject(); - if (vmobject.is_shared_by_multiple_regions()) { - vmobject.for_each_region([&](auto& region) { - if (!region.do_remap_vmobject_page_range(page_index, page_count)) - success = false; - }); - } else { - if (!do_remap_vmobject_page_range(page_index, page_count)) - success = false; - } - return success; -} - bool Region::do_remap_vmobject_page(size_t page_index, bool with_flush) { ScopedSpinLock lock(vmobject().m_lock); @@ -428,7 +322,7 @@ PageFaultResponse Region::handle_fault(PageFault const& fault) if (page_slot->is_lazy_committed_page()) { auto page_index_in_vmobject = translate_to_vmobject_page(page_index_in_region); VERIFY(m_vmobject->is_anonymous()); - page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page({}, page_index_in_vmobject); + page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page({}); remap_vmobject_page(page_index_in_vmobject); return PageFaultResponse::Continue; } @@ -472,7 +366,7 @@ PageFaultResponse Region::handle_zero_fault(size_t page_index_in_region) if (page_slot->is_lazy_committed_page()) { VERIFY(m_vmobject->is_anonymous()); - page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page({}, page_index_in_vmobject); + page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page({}); dbgln_if(PAGE_FAULT_DEBUG, " >> ALLOCATED COMMITTED {}", page_slot->paddr()); } else { page_slot = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes); diff --git a/Kernel/VM/Region.h b/Kernel/VM/Region.h index 34e76802bc..af680e6185 100644 --- a/Kernel/VM/Region.h +++ b/Kernel/VM/Region.h @@ -17,7 +17,6 @@ #include <Kernel/Sections.h> #include <Kernel/UnixTypes.h> #include <Kernel/VM/PageFaultResponse.h> -#include <Kernel/VM/PurgeablePageRanges.h> #include <Kernel/VM/RangeAllocator.h> namespace Kernel { @@ -28,8 +27,7 @@ enum class ShouldFlushTLB { }; class Region final - : public Weakable<Region> - , public PurgeablePageRanges { + : public Weakable<Region> { friend class MemoryManager; MAKE_SLAB_ALLOCATED(Region) @@ -201,15 +199,11 @@ public: void remap(); - bool remap_vmobject_page_range(size_t page_index, size_t page_count); - - bool is_volatile(VirtualAddress vaddr, size_t size) const; enum class SetVolatileError { Success = 0, NotPurgeable, OutOfMemory }; - SetVolatileError set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged); RefPtr<Process> get_owner(); @@ -219,7 +213,8 @@ public: private: Region(Range const&, NonnullRefPtr<VMObject>, size_t offset_in_vmobject, OwnPtr<KString>, Region::Access access, Cacheable, bool shared); - bool do_remap_vmobject_page_range(size_t page_index, size_t page_count); + bool remap_vmobject_page(size_t page_index, bool with_flush = true); + bool do_remap_vmobject_page(size_t page_index, bool with_flush = true); void set_access_bit(Access access, bool b) { @@ -229,18 +224,12 @@ private: m_access &= ~access; } - bool do_remap_vmobject_page(size_t index, bool with_flush = true); - bool remap_vmobject_page(size_t index, bool with_flush = true); - PageFaultResponse handle_cow_fault(size_t page_index); PageFaultResponse handle_inode_fault(size_t page_index); PageFaultResponse handle_zero_fault(size_t page_index); bool map_individual_page_impl(size_t page_index); - void register_purgeable_page_ranges(); - void unregister_purgeable_page_ranges(); - RefPtr<PageDirectory> m_page_directory; Range m_range; size_t m_offset_in_vmobject { 0 }; diff --git a/Kernel/VM/Space.cpp b/Kernel/VM/Space.cpp index 28dc3ee12f..c116285c2b 100644 --- a/Kernel/VM/Space.cpp +++ b/Kernel/VM/Space.cpp @@ -414,7 +414,10 @@ size_t Space::amount_purgeable_volatile() const ScopedSpinLock lock(m_lock); size_t amount = 0; for (auto& region : m_regions) { - if (region->vmobject().is_anonymous() && static_cast<const AnonymousVMObject&>(region->vmobject()).is_any_volatile()) + if (!region->vmobject().is_anonymous()) + continue; + auto const& vmobject = static_cast<AnonymousVMObject const&>(region->vmobject()); + if (vmobject.is_purgeable() && vmobject.is_volatile()) amount += region->amount_resident(); } return amount; @@ -425,7 +428,10 @@ size_t Space::amount_purgeable_nonvolatile() const ScopedSpinLock lock(m_lock); size_t amount = 0; for (auto& region : m_regions) { - if (region->vmobject().is_anonymous() && !static_cast<const AnonymousVMObject&>(region->vmobject()).is_any_volatile()) + if (!region->vmobject().is_anonymous()) + continue; + auto const& vmobject = static_cast<AnonymousVMObject const&>(region->vmobject()); + if (vmobject.is_purgeable() && !vmobject.is_volatile()) amount += region->amount_resident(); } return amount; diff --git a/Userland/Libraries/LibC/malloc.cpp b/Userland/Libraries/LibC/malloc.cpp index 9c412c55cf..086c86c411 100644 --- a/Userland/Libraries/LibC/malloc.cpp +++ b/Userland/Libraries/LibC/malloc.cpp @@ -158,7 +158,7 @@ extern "C" { static void* os_alloc(size_t size, const char* name) { - auto* ptr = serenity_mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0, ChunkedBlock::block_size, name); + auto* ptr = serenity_mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_PURGEABLE, 0, 0, ChunkedBlock::block_size, name); VERIFY(ptr != MAP_FAILED); return ptr; } diff --git a/Userland/Libraries/LibC/sys/mman.h b/Userland/Libraries/LibC/sys/mman.h index d2638bd03f..be135ab93d 100644 --- a/Userland/Libraries/LibC/sys/mman.h +++ b/Userland/Libraries/LibC/sys/mman.h @@ -18,6 +18,7 @@ #define MAP_STACK 0x40 #define MAP_NORESERVE 0x80 #define MAP_RANDOMIZED 0x100 +#define MAP_PURGEABLE 0x200 #define PROT_READ 0x1 #define PROT_WRITE 0x2 diff --git a/Userland/Libraries/LibGfx/Bitmap.cpp b/Userland/Libraries/LibGfx/Bitmap.cpp index 8d8f4b87b7..da121366f0 100644 --- a/Userland/Libraries/LibGfx/Bitmap.cpp +++ b/Userland/Libraries/LibGfx/Bitmap.cpp @@ -544,11 +544,11 @@ void Bitmap::set_volatile() int rc = madvise(m_data, size_in_bytes(), MADV_SET_NONVOLATILE); if (rc < 0) { if (errno == ENOMEM) { - was_purged = was_purged_int; + was_purged = true; return false; } - perror("madvise(MADV_SET_NONVOLATILE)"); + VERIFY_NOT_REACHED(); } was_purged = rc != 0; #endif @@ -574,6 +574,7 @@ Optional<BackingStore> Bitmap::try_allocate_backing_store(BitmapFormat format, I int map_flags = MAP_ANONYMOUS | MAP_PRIVATE; #ifdef __serenity__ + map_flags |= MAP_PURGEABLE; void* data = mmap_with_name(nullptr, data_size_in_bytes, PROT_READ | PROT_WRITE, map_flags, 0, 0, String::formatted("GraphicsBitmap [{}]", size).characters()); #else void* data = mmap(nullptr, data_size_in_bytes, PROT_READ | PROT_WRITE, map_flags, 0, 0); |