Kernel: Make SharedInodeVMObject allocation OOM-safe

4 files changed, 17 insertions, 6 deletions

diff --git a/Kernel/FileSystem/InodeFile.cpp b/Kernel/FileSystem/InodeFile.cpp
index 9e4ab77c52..ed051a7416 100644
--- a/Kernel/FileSystem/InodeFile.cpp
+++ b/Kernel/FileSystem/InodeFile.cpp
@@ -97,7 +97,7 @@ KResultOr<Region*> InodeFile::mmap(Process& process, FileDescription& descriptio
     // FIXME: If PROT_EXEC, check that the underlying file system isn't mounted noexec.
     RefPtr<InodeVMObject> vmobject;
     if (shared)
-        vmobject = SharedInodeVMObject::create_with_inode(inode());
+        vmobject = SharedInodeVMObject::try_create_with_inode(inode());
     else
         vmobject = PrivateInodeVMObject::create_with_inode(inode());
     if (!vmobject)
diff --git a/Kernel/Syscalls/execve.cpp b/Kernel/Syscalls/execve.cpp
index 6c8b93424c..3c79f34069 100644
--- a/Kernel/Syscalls/execve.cpp
+++ b/Kernel/Syscalls/execve.cpp
@@ -162,7 +162,11 @@ struct RequiredLoadRange {
 static KResultOr<RequiredLoadRange> get_required_load_range(FileDescription& program_description)
 {
     auto& inode = *(program_description.inode());
-    auto vmobject = SharedInodeVMObject::create_with_inode(inode);
+    auto vmobject = SharedInodeVMObject::try_create_with_inode(inode);
+    if (!vmobject) {
+        dbgln("get_required_load_range: Unable to allocate SharedInodeVMObject");
+        return ENOMEM;
+    }
 
     size_t executable_size = inode.size();
 
@@ -263,7 +267,12 @@ static KResultOr<LoadResult> load_elf_object(NonnullOwnPtr<Space> new_space, Fil
     FlatPtr load_offset, ShouldAllocateTls should_allocate_tls, ShouldAllowSyscalls should_allow_syscalls)
 {
     auto& inode = *(object_description.inode());
-    auto vmobject = SharedInodeVMObject::create_with_inode(inode);
+    auto vmobject = SharedInodeVMObject::try_create_with_inode(inode);
+    if (!vmobject) {
+        dbgln("load_elf_object: Unable to allocate SharedInodeVMObject");
+        return ENOMEM;
+    }
+
     if (vmobject->writable_mappings()) {
         dbgln("Refusing to execute a write-mapped program");
         return ETXTBSY;
diff --git a/Kernel/VM/SharedInodeVMObject.cpp b/Kernel/VM/SharedInodeVMObject.cpp
index 91e7873fb7..383e35f254 100644
--- a/Kernel/VM/SharedInodeVMObject.cpp
+++ b/Kernel/VM/SharedInodeVMObject.cpp
@@ -9,12 +9,14 @@
 
 namespace Kernel {
 
-NonnullRefPtr<SharedInodeVMObject> SharedInodeVMObject::create_with_inode(Inode& inode)
+RefPtr<SharedInodeVMObject> SharedInodeVMObject::try_create_with_inode(Inode& inode)
 {
     size_t size = inode.size();
     if (auto shared_vmobject = inode.shared_vmobject())
         return shared_vmobject.release_nonnull();
-    auto vmobject = adopt_ref(*new SharedInodeVMObject(inode, size));
+    auto vmobject = adopt_ref_if_nonnull(new (nothrow) SharedInodeVMObject(inode, size));
+    if (!vmobject)
+        return nullptr;
     vmobject->inode().set_shared_vmobject(*vmobject);
     return vmobject;
 }
diff --git a/Kernel/VM/SharedInodeVMObject.h b/Kernel/VM/SharedInodeVMObject.h
index a0e45eb0c9..98e2a30082 100644
--- a/Kernel/VM/SharedInodeVMObject.h
+++ b/Kernel/VM/SharedInodeVMObject.h
@@ -16,7 +16,7 @@ class SharedInodeVMObject final : public InodeVMObject {
     AK_MAKE_NONMOVABLE(SharedInodeVMObject);
 
 public:
-    static NonnullRefPtr<SharedInodeVMObject> create_with_inode(Inode&);
+    static RefPtr<SharedInodeVMObject> try_create_with_inode(Inode&);
     virtual RefPtr<VMObject> clone() override;
 
 private: