/* * Copyright (c) 2020, Liav A. * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include #include #include #include #include #include namespace Kernel::PCI { #define PCI_MMIO_CONFIG_SPACE_SIZE 4096 static Access* s_access; Access& Access::the() { if (s_access == nullptr) { VERIFY_NOT_REACHED(); // We failed to initialize the PCI subsystem, so stop here! } return *s_access; } bool Access::is_initialized() { return (s_access != nullptr); } UNMAP_AFTER_INIT bool Access::initialize_for_memory_access(PhysicalAddress mcfg_table) { if (Access::is_initialized()) return false; InterruptDisabler disabler; auto* access = new Access(Access::AccessType::Memory); if (!access->scan_pci_domains(mcfg_table)) return false; access->rescan_hardware(); dbgln_if(PCI_DEBUG, "PCI: MMIO access initialised."); return true; } UNMAP_AFTER_INIT bool Access::scan_pci_domains(PhysicalAddress mcfg_table) { auto checkup_region_or_error = MM.allocate_kernel_region(mcfg_table.page_base(), (PAGE_SIZE * 2), "PCI MCFG Checkup", Memory::Region::Access::ReadWrite); if (checkup_region_or_error.is_error()) return false; dbgln_if(PCI_DEBUG, "PCI: Checking MCFG Table length to choose the correct mapping size"); auto* sdt = (ACPI::Structures::SDTHeader*)checkup_region_or_error.value()->vaddr().offset(mcfg_table.offset_in_page()).as_ptr(); u32 length = sdt->length; u8 revision = sdt->revision; dbgln("PCI: MCFG, length: {}, revision: {}", length, revision); auto mcfg_region_or_error = MM.allocate_kernel_region(mcfg_table.page_base(), Memory::page_round_up(length) + PAGE_SIZE, "PCI Parsing MCFG", Memory::Region::Access::ReadWrite); if (mcfg_region_or_error.is_error()) return false; auto& mcfg = *(ACPI::Structures::MCFG*)mcfg_region_or_error.value()->vaddr().offset(mcfg_table.offset_in_page()).as_ptr(); dbgln_if(PCI_DEBUG, "PCI: Checking MCFG @ {}, {}", VirtualAddress(&mcfg), mcfg_table); for (u32 index = 0; index < ((mcfg.header.length - sizeof(ACPI::Structures::MCFG)) / sizeof(ACPI::Structures::PCI_MMIO_Descriptor)); index++) { u8 start_bus = mcfg.descriptors[index].start_pci_bus; u8 end_bus = mcfg.descriptors[index].end_pci_bus; u32 lower_addr = mcfg.descriptors[index].base_addr; auto result = m_domains.set(index, { PhysicalAddress(lower_addr), start_bus, end_bus }); VERIFY(result == AK::HashSetResult::InsertedNewEntry); dmesgln("PCI: New PCI domain @ {}, PCI buses ({}-{})", PhysicalAddress { lower_addr }, start_bus, end_bus); } VERIFY(m_domains.contains(0)); dmesgln("PCI: MMIO domain: {}", m_domains.size()); return true; } UNMAP_AFTER_INIT bool Access::initialize_for_io_access() { if (Access::is_initialized()) { return false; } auto* access = new Access(Access::AccessType::IO); access->rescan_hardware(); dbgln_if(PCI_DEBUG, "PCI: IO access initialised."); return true; } UNMAP_AFTER_INIT Access::Access(AccessType access_type) : m_enumerated_buses(256, false) , m_access_type(access_type) { if (access_type == AccessType::IO) dmesgln("PCI: Using I/O instructions for PCI configuration space access"); else dmesgln("PCI: Using memory access for PCI configuration space accesses"); s_access = this; } Optional Access::determine_memory_mapped_bus_base_address(u32 domain, u8 bus) const { auto chosen_domain = m_domains.get(domain); if (!chosen_domain.has_value()) return {}; if (!(chosen_domain.value().start_bus() <= bus && bus <= chosen_domain.value().end_bus())) return {}; return chosen_domain.value().paddr().offset(memory_range_per_bus * (bus - chosen_domain.value().start_bus())); } void Access::map_bus_region(u32 domain, u8 bus) { VERIFY(m_access_lock.is_locked()); if (m_mapped_bus == bus && m_mapped_bus_region) return; auto bus_base_address = determine_memory_mapped_bus_base_address(domain, bus); // FIXME: Find a way to propagate error from here. if (!bus_base_address.has_value()) VERIFY_NOT_REACHED(); auto region_or_error = MM.allocate_kernel_region(bus_base_address.value(), memory_range_per_bus, "PCI ECAM", Memory::Region::Access::ReadWrite); // FIXME: Find a way to propagate error from here. if (region_or_error.is_error()) VERIFY_NOT_REACHED(); m_mapped_bus_region = region_or_error.release_value(); m_mapped_bus = bus; dbgln_if(PCI_DEBUG, "PCI: New PCI ECAM Mapped region for bus {} @ {} {}", bus, m_mapped_bus_region->vaddr(), m_mapped_bus_region->physical_page(0)->paddr()); } VirtualAddress Access::get_device_configuration_memory_mapped_space(Address address) { VERIFY(m_access_lock.is_locked()); dbgln_if(PCI_DEBUG, "PCI: Getting device configuration space for {}", address); map_bus_region(address.domain(), address.bus()); return m_mapped_bus_region->vaddr().offset(mmio_device_space_size * address.function() + (mmio_device_space_size * to_underlying(Limits::MaxFunctionsPerDevice)) * address.device()); } u8 Access::io_read8_field(Address address, u32 field) { MutexLocker lock(m_access_lock); dbgln_if(PCI_DEBUG, "PCI: IO Reading 8-bit field {:#08x} for {}", field, address); IO::out32(PCI::address_port, address.io_address_for_field(field)); return IO::in8(PCI::value_port + (field & 3)); } u16 Access::io_read16_field(Address address, u32 field) { MutexLocker lock(m_access_lock); dbgln_if(PCI_DEBUG, "PCI: IO Reading 16-bit field {:#08x} for {}", field, address); IO::out32(PCI::address_port, address.io_address_for_field(field)); return IO::in16(PCI::value_port + (field & 2)); } u32 Access::io_read32_field(Address address, u32 field) { MutexLocker lock(m_access_lock); dbgln_if(PCI_DEBUG, "PCI: IO Reading 32-bit field {:#08x} for {}", field, address); IO::out32(PCI::address_port, address.io_address_for_field(field)); return IO::in32(PCI::value_port); } void Access::io_write8_field(Address address, u32 field, u8 value) { MutexLocker lock(m_access_lock); dbgln_if(PCI_DEBUG, "PCI: IO Writing to 8-bit field {:#08x}, value={:#02x} for {}", field, value, address); IO::out32(PCI::address_port, address.io_address_for_field(field)); IO::out8(PCI::value_port + (field & 3), value); } void Access::io_write16_field(Address address, u32 field, u16 value) { MutexLocker lock(m_access_lock); dbgln_if(PCI_DEBUG, "PCI: IO Writing to 16-bit field {:#08x}, value={:#02x} for {}", field, value, address); IO::out32(PCI::address_port, address.io_address_for_field(field)); IO::out16(PCI::value_port + (field & 2), value); } void Access::io_write32_field(Address address, u32 field, u32 value) { MutexLocker lock(m_access_lock); dbgln_if(PCI_DEBUG, "PCI: IO Writing to 32-bit field {:#08x}, value={:#02x} for {}", field, value, address); IO::out32(PCI::address_port, address.io_address_for_field(field)); IO::out32(PCI::value_port, value); } u8 Access::memory_read8_field(Address address, u32 field) { MutexLocker lock(m_access_lock); VERIFY(field <= 0xfff); dbgln_if(PCI_DEBUG, "PCI: MMIO Reading 8-bit field {:#08x} for {}", field, address); return *((volatile u8*)(get_device_configuration_memory_mapped_space(address).get() + (field & 0xfff))); } u16 Access::memory_read16_field(Address address, u32 field) { MutexLocker lock(m_access_lock); VERIFY(field < 0xfff); dbgln_if(PCI_DEBUG, "PCI: MMIO Reading 16-bit field {:#08x} for {}", field, address); u16 data = 0; ByteReader::load(get_device_configuration_memory_mapped_space(address).offset(field & 0xfff).as_ptr(), data); return data; } u32 Access::memory_read32_field(Address address, u32 field) { MutexLocker lock(m_access_lock); VERIFY(field <= 0xffc); dbgln_if(PCI_DEBUG, "PCI: MMIO Reading 32-bit field {:#08x} for {}", field, address); u32 data = 0; ByteReader::load(get_device_configuration_memory_mapped_space(address).offset(field & 0xfff).as_ptr(), data); return data; } void Access::memory_write8_field(Address address, u32 field, u8 value) { MutexLocker lock(m_access_lock); VERIFY(field <= 0xfff); dbgln_if(PCI_DEBUG, "PCI: MMIO Writing 8-bit field {:#08x}, value={:#02x} for {}", field, value, address); *((volatile u8*)(get_device_configuration_memory_mapped_space(address).get() + (field & 0xfff))) = value; } void Access::memory_write16_field(Address address, u32 field, u16 value) { MutexLocker lock(m_access_lock); VERIFY(field < 0xfff); dbgln_if(PCI_DEBUG, "PCI: MMIO Writing 16-bit field {:#08x}, value={:#02x} for {}", field, value, address); ByteReader::store(get_device_configuration_memory_mapped_space(address).offset(field & 0xfff).as_ptr(), value); } void Access::memory_write32_field(Address address, u32 field, u32 value) { MutexLocker lock(m_access_lock); VERIFY(field <= 0xffc); dbgln_if(PCI_DEBUG, "PCI: MMIO Writing 32-bit field {:#08x}, value={:#02x} for {}", field, value, address); ByteReader::store(get_device_configuration_memory_mapped_space(address).offset(field & 0xfff).as_ptr(), value); } void Access::write8_field(Address address, u32 field, u8 value) { switch (m_access_type) { case AccessType::IO: io_write8_field(address, field, value); return; case AccessType::Memory: memory_write8_field(address, field, value); return; } VERIFY_NOT_REACHED(); } void Access::write16_field(Address address, u32 field, u16 value) { switch (m_access_type) { case AccessType::IO: io_write16_field(address, field, value); return; case AccessType::Memory: memory_write16_field(address, field, value); return; } VERIFY_NOT_REACHED(); } void Access::write32_field(Address address, u32 field, u32 value) { switch (m_access_type) { case AccessType::IO: io_write32_field(address, field, value); return; case AccessType::Memory: memory_write32_field(address, field, value); return; } VERIFY_NOT_REACHED(); } u8 Access::read8_field(Address address, RegisterOffset field) { return read8_field(address, to_underlying(field)); } u16 Access::read16_field(Address address, RegisterOffset field) { return read16_field(address, to_underlying(field)); } u8 Access::read8_field(Address address, u32 field) { switch (m_access_type) { case AccessType::IO: return io_read8_field(address, field); case AccessType::Memory: return memory_read8_field(address, field); } VERIFY_NOT_REACHED(); } u16 Access::read16_field(Address address, u32 field) { switch (m_access_type) { case AccessType::IO: return io_read16_field(address, field); case AccessType::Memory: return memory_read16_field(address, field); } VERIFY_NOT_REACHED(); } u32 Access::read32_field(Address address, u32 field) { switch (m_access_type) { case AccessType::IO: return io_read32_field(address, field); case AccessType::Memory: return memory_read32_field(address, field); } VERIFY_NOT_REACHED(); } UNMAP_AFTER_INIT void Access::rescan_hardware() { MutexLocker locker(m_scan_lock); VERIFY(m_device_identifiers.is_empty()); if (m_access_type == AccessType::IO) { dbgln_if(PCI_DEBUG, "PCI: IO enumerating hardware"); // First scan bus 0. Find any device on that bus, and if it's a PCI-to-PCI // bridge, recursively scan it too. m_enumerated_buses.set(0, true); enumerate_bus(-1, 0, true); // Handle Multiple PCI host bridges on slot 0, device 0. // If we happen to miss some PCI buses because they are not reachable through // recursive PCI-to-PCI bridges starting from bus 0, we might find them here. if ((read8_field(Address(), PCI::RegisterOffset::HEADER_TYPE) & 0x80) != 0) { for (int bus = 1; bus < 256; ++bus) { if (read16_field(Address(0, 0, 0, bus), PCI::RegisterOffset::VENDOR_ID) == PCI::none_value) continue; if (read16_field(Address(0, 0, 0, bus), PCI::RegisterOffset::CLASS) != 0x6) continue; if (m_enumerated_buses.get(bus)) continue; enumerate_bus(-1, bus, false); m_enumerated_buses.set(bus, true); } } return; } VERIFY(m_access_type == AccessType::Memory); for (u32 domain = 0; domain < m_domains.size(); domain++) { dbgln_if(PCI_DEBUG, "PCI: Scan memory mapped domain {}", domain); // Single PCI host controller. if ((read8_field(Address(domain), PCI::RegisterOffset::HEADER_TYPE) & 0x80) == 0) { enumerate_bus(-1, 0, true); return; } // Multiple PCI host controllers. for (u8 function = 0; function < 8; ++function) { if (read16_field(Address(domain, 0, 0, function), PCI::RegisterOffset::VENDOR_ID) == PCI::none_value) break; enumerate_bus(-1, function, false); } } } UNMAP_AFTER_INIT Optional Access::get_capabilities_pointer(Address address) { dbgln_if(PCI_DEBUG, "PCI: Getting capabilities pointer for {}", address); if (read16_field(address, PCI::RegisterOffset::STATUS) & (1 << 4)) { dbgln_if(PCI_DEBUG, "PCI: Found capabilities pointer for {}", address); return read8_field(address, PCI::RegisterOffset::CAPABILITIES_POINTER); } dbgln_if(PCI_DEBUG, "PCI: No capabilities pointer for {}", address); return {}; } UNMAP_AFTER_INIT Vector Access::get_capabilities(Address address) { dbgln_if(PCI_DEBUG, "PCI: Getting capabilities for {}", address); auto capabilities_pointer = get_capabilities_pointer(address); if (!capabilities_pointer.has_value()) { dbgln_if(PCI_DEBUG, "PCI: No capabilities for {}", address); return {}; } Vector capabilities; auto capability_pointer = capabilities_pointer.value(); while (capability_pointer != 0) { dbgln_if(PCI_DEBUG, "PCI: Reading in capability at {:#02x} for {}", capability_pointer, address); u16 capability_header = read16_field(address, capability_pointer); u8 capability_id = capability_header & 0xff; capabilities.append({ address, capability_id, capability_pointer }); capability_pointer = capability_header >> 8; } return capabilities; } UNMAP_AFTER_INIT void Access::enumerate_functions(int type, u8 bus, u8 device, u8 function, bool recursive) { dbgln_if(PCI_DEBUG, "PCI: Enumerating function type={}, bus={}, device={}, function={}", type, bus, device, function); Address address(0, bus, device, function); auto read_type = (read8_field(address, PCI::RegisterOffset::CLASS) << 8u) | read8_field(address, PCI::RegisterOffset::SUBCLASS); if (type == -1 || type == read_type) { HardwareID id = { read16_field(address, PCI::RegisterOffset::VENDOR_ID), read16_field(address, PCI::RegisterOffset::DEVICE_ID) }; ClassCode class_code = read8_field(address, PCI::RegisterOffset::CLASS); SubclassCode subclass_code = read8_field(address, PCI::RegisterOffset::SUBCLASS); ProgrammingInterface prog_if = read8_field(address, PCI::RegisterOffset::PROG_IF); RevisionID revision_id = read8_field(address, PCI::RegisterOffset::REVISION_ID); SubsystemID subsystem_id = read16_field(address, PCI::RegisterOffset::SUBSYSTEM_ID); SubsystemVendorID subsystem_vendor_id = read16_field(address, PCI::RegisterOffset::SUBSYSTEM_VENDOR_ID); InterruptLine interrupt_line = read8_field(address, PCI::RegisterOffset::INTERRUPT_LINE); InterruptPin interrupt_pin = read8_field(address, PCI::RegisterOffset::INTERRUPT_PIN); m_device_identifiers.append(DeviceIdentifier { address, id, revision_id, class_code, subclass_code, prog_if, subsystem_id, subsystem_vendor_id, interrupt_line, interrupt_pin, get_capabilities(address) }); } if (read_type == (to_underlying(PCI::ClassID::Bridge) << 8 | to_underlying(PCI::Bridge::SubclassID::PCI_TO_PCI)) && recursive && (!m_enumerated_buses.get(read8_field(address, PCI::RegisterOffset::SECONDARY_BUS)))) { u8 secondary_bus = read8_field(address, PCI::RegisterOffset::SECONDARY_BUS); dbgln_if(PCI_DEBUG, "PCI: Found secondary bus: {}", secondary_bus); VERIFY(secondary_bus != bus); m_enumerated_buses.set(secondary_bus, true); enumerate_bus(type, secondary_bus, recursive); } } UNMAP_AFTER_INIT void Access::enumerate_device(int type, u8 bus, u8 device, bool recursive) { dbgln_if(PCI_DEBUG, "PCI: Enumerating device type={}, bus={}, device={}", type, bus, device); Address address(0, bus, device, 0); if (read16_field(address, PCI::RegisterOffset::VENDOR_ID) == PCI::none_value) return; enumerate_functions(type, bus, device, 0, recursive); if (!(read8_field(address, PCI::RegisterOffset::HEADER_TYPE) & 0x80)) return; for (u8 function = 1; function < 8; ++function) { Address address(0, bus, device, function); if (read16_field(address, PCI::RegisterOffset::VENDOR_ID) != PCI::none_value) enumerate_functions(type, bus, device, function, recursive); } } UNMAP_AFTER_INIT void Access::enumerate_bus(int type, u8 bus, bool recursive) { dbgln_if(PCI_DEBUG, "PCI: Enumerating bus type={}, bus={}", type, bus); for (u8 device = 0; device < 32; ++device) enumerate_device(type, bus, device, recursive); } void Access::fast_enumerate(Function& callback) const { MutexLocker locker(m_scan_lock); VERIFY(!m_device_identifiers.is_empty()); for (auto& device_identifier : m_device_identifiers) { callback(device_identifier); } } DeviceIdentifier Access::get_device_identifier(Address address) const { for (auto device_identifier : m_device_identifiers) { if (device_identifier.address().domain() == address.domain() && device_identifier.address().bus() == address.bus() && device_identifier.address().device() == address.device() && device_identifier.address().function() == address.function()) { return device_identifier; } } VERIFY_NOT_REACHED(); } }