From 9b3b0531e5cd4054134f1f0ae8e2303ffe33903f Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Sat, 29 Apr 2023 22:27:33 +0200 Subject: Kernel: Add MSIx support to NVMe Add MSIx support to NVMe. Prefer MSIx over pin-based interrupts as they are more efficient and all modern hardware support them. --- Kernel/Storage/NVMe/NVMeController.cpp | 16 +++++++++++----- Kernel/Storage/NVMe/NVMeController.h | 1 + Kernel/Storage/NVMe/NVMeInterruptQueue.cpp | 4 ++-- Kernel/Storage/NVMe/NVMeInterruptQueue.h | 4 +++- 4 files changed, 17 insertions(+), 8 deletions(-) (limited to 'Kernel') diff --git a/Kernel/Storage/NVMe/NVMeController.cpp b/Kernel/Storage/NVMe/NVMeController.cpp index b01591aad2..181cec57da 100644 --- a/Kernel/Storage/NVMe/NVMeController.cpp +++ b/Kernel/Storage/NVMe/NVMeController.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,9 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::initialize(bool is_queue_polled) m_ready_timeout = Time::from_milliseconds((CAP_TO(caps) + 1) * 500); // CAP.TO is in 500ms units calculate_doorbell_stride(); + // IO queues + 1 admin queue + m_irq_type = TRY(reserve_irqs(nr_of_queues + 1, true)); + TRY(create_admin_queue(queue_type)); VERIFY(m_admin_queue_ready == true); @@ -281,13 +285,15 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(QueueType queu m_controller_regs->acq = reinterpret_cast(AK::convert_between_host_and_little_endian(cq_dma_pages.first()->paddr().as_ptr())); m_controller_regs->asq = reinterpret_cast(AK::convert_between_host_and_little_endian(sq_dma_pages.first()->paddr().as_ptr())); + auto irq = TRY(allocate_irq(0)); // Admin queue always uses the 0th index when using MSIx + maybe_error = start_controller(); if (maybe_error.is_error()) { dmesgln_pci(*this, "Failed to restart the NVMe controller"); return maybe_error; } set_admin_queue_ready_flag(); - m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, device_identifier().interrupt_line().value(), qdepth, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type)); + m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, irq, qdepth, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type)); dbgln_if(NVME_DEBUG, "NVMe: Admin queue created"); return {}; @@ -325,9 +331,8 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 qid, QueueType sub.create_cq.qsize = AK::convert_between_host_and_little_endian(IO_QUEUE_SIZE - 1); auto flags = (queue_type == QueueType::IRQ) ? QUEUE_IRQ_ENABLED : QUEUE_IRQ_DISABLED; flags |= QUEUE_PHY_CONTIGUOUS; - // TODO: Eventually move to MSI. - // For now using pin based interrupts. Clear the first 16 bits - // to use pin-based interrupts. + // When using MSIx interrupts, qid is used as an index into the interrupt table + sub.create_cq.irq_vector = (m_irq_type == PCI::InterruptType::PIN) ? 0 : qid; sub.create_cq.cq_flags = AK::convert_between_host_and_little_endian(flags & 0xFFFF); submit_admin_command(sub, true); } @@ -346,8 +351,9 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 qid, QueueType auto queue_doorbell_offset = REG_SQ0TDBL_START + ((2 * qid) * (4 << m_dbl_stride)); auto doorbell_regs = TRY(Memory::map_typed_writable(PhysicalAddress(m_bar + queue_doorbell_offset))); + auto irq = TRY(allocate_irq(qid)); - m_queues.append(TRY(NVMeQueue::try_create(*this, qid, device_identifier().interrupt_line().value(), IO_QUEUE_SIZE, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type))); + m_queues.append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type))); dbgln_if(NVME_DEBUG, "NVMe: Created IO Queue with QID{}", m_queues.size()); return {}; } diff --git a/Kernel/Storage/NVMe/NVMeController.h b/Kernel/Storage/NVMe/NVMeController.h index 12b5db9796..50c8428ec7 100644 --- a/Kernel/Storage/NVMe/NVMeController.h +++ b/Kernel/Storage/NVMe/NVMeController.h @@ -77,6 +77,7 @@ private: AK::Time m_ready_timeout; u32 m_bar { 0 }; u8 m_dbl_stride { 0 }; + PCI::InterruptType m_irq_type; QueueType m_queue_type { QueueType::IRQ }; static Atomic s_controller_id; }; diff --git a/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp b/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp index 289b37b9c4..353673fb10 100644 --- a/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp +++ b/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp @@ -11,9 +11,9 @@ namespace Kernel { -UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue([[maybe_unused]] PCI::Device& device, NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, Vector> cq_dma_page, OwnPtr sq_dma_region, Vector> sq_dma_page, Memory::TypedMapping db_regs) +UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, Vector> cq_dma_page, OwnPtr sq_dma_region, Vector> sq_dma_page, Memory::TypedMapping db_regs) : NVMeQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs)) - , IRQHandler(irq) + , PCIIRQHandler(device, irq) { enable_irq(); } diff --git a/Kernel/Storage/NVMe/NVMeInterruptQueue.h b/Kernel/Storage/NVMe/NVMeInterruptQueue.h index c6e80603e8..6cfd4fccd4 100644 --- a/Kernel/Storage/NVMe/NVMeInterruptQueue.h +++ b/Kernel/Storage/NVMe/NVMeInterruptQueue.h @@ -6,16 +6,18 @@ #pragma once +#include #include namespace Kernel { class NVMeInterruptQueue : public NVMeQueue - , public IRQHandler { + , public PCIIRQHandler { public: NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, Vector> cq_dma_page, OwnPtr sq_dma_region, Vector> sq_dma_page, Memory::TypedMapping db_regs); void submit_sqe(NVMeSubmission& submission) override; virtual ~NVMeInterruptQueue() override {}; + virtual StringView purpose() const override { return "NVMe"sv; }; private: virtual void complete_current_request(u16 cmdid, u16 status) override; -- cgit v1.2.3