diff options
author | Jakub V. Flasar <38976370+Kubiisek@users.noreply.github.com> | 2022-03-08 18:21:40 +0100 |
---|---|---|
committer | Brian Gianforcaro <b.gianfo@gmail.com> | 2022-03-12 14:54:12 -0800 |
commit | 6d2c298b66a9dedbb0c32623e2aef187edce77b3 (patch) | |
tree | de441af6769da016638632af14d472568e210915 /Kernel/Arch/aarch64 | |
parent | f94293f1216f4843dcba87c8809f17cb619e154f (diff) | |
download | serenity-6d2c298b66a9dedbb0c32623e2aef187edce77b3.zip |
Kernel: Move aarch64 Prekernel into Kernel
As there is no need for a Prekernel on aarch64, the Prekernel code was
moved into Kernel itself. The functionality remains the same.
SERENITY_KERNEL_AND_INITRD in run.sh specifies a kernel and an inital
ramdisk to be used by the emulator. This is needed because aarch64
does not need a Prekernel and the other ones do.
Diffstat (limited to 'Kernel/Arch/aarch64')
30 files changed, 2126 insertions, 0 deletions
diff --git a/Kernel/Arch/aarch64/Aarch64_asm_utils.S b/Kernel/Arch/aarch64/Aarch64_asm_utils.S new file mode 100644 index 0000000000..33f2d4de7f --- /dev/null +++ b/Kernel/Arch/aarch64/Aarch64_asm_utils.S @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * Copyright (c) 2021, Jesse Buhagiar <jooster669@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +.global wait_cycles +.type wait_cycles, @function +wait_cycles: +Lstart: + // This is probably too fast when caching and branch prediction is turned on. + // FIXME: Make timer-based. + subs x0, x0, #1 + bne Lstart + ret + +.global enter_el2_from_el3 +.type enter_el2_from_el3, @function +enter_el2_from_el3: + adr x0, entered_el2 + msr elr_el3, x0 + eret +entered_el2: + ret + +.global enter_el1_from_el2 +.type enter_el1_from_el2, @function +enter_el1_from_el2: + adr x0, entered_el1 + msr elr_el2, x0 + eret +entered_el1: + ret + +// +// Installs the EL1 vector table +// Args: +// x0 - Address of vector table +// +// This function doesn't return a value +// +.global el1_vector_table_install +.type el1_vector_table_install, @function +el1_vector_table_install: + msr VBAR_EL1, x0 + ret diff --git a/Kernel/Arch/aarch64/Aarch64_asm_utils.h b/Kernel/Arch/aarch64/Aarch64_asm_utils.h new file mode 100644 index 0000000000..5536904b2b --- /dev/null +++ b/Kernel/Arch/aarch64/Aarch64_asm_utils.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +extern "C" void wait_cycles(int n); +extern "C" void el1_vector_table_install(void* vector_table); + +// CPU initialization functions +extern "C" [[noreturn]] void return_from_el2(); +extern "C" [[noreturn]] void return_from_el3(); diff --git a/Kernel/Arch/aarch64/BootPPMParser.cpp b/Kernel/Arch/aarch64/BootPPMParser.cpp new file mode 100644 index 0000000000..6061e1c427 --- /dev/null +++ b/Kernel/Arch/aarch64/BootPPMParser.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "BootPPMParser.h" + +namespace Prekernel { + +BootPPMParser::BootPPMParser(u8 const* buffer, u32 buffer_size) +{ + m_cursor = reinterpret_cast<char const*>(buffer); + m_buffer_end = m_cursor + buffer_size; +} + +bool BootPPMParser::parse() +{ + if (!check_position()) { + return false; + } + if (!parse_magic()) { + return false; + } + if (!parse_new_line()) { + return false; + } + if (!parse_comment()) { + return false; + } + if (!parse_integer(image.width)) { + return false; + } + if (!parse_integer(image.height)) { + return false; + } + u32 max_color_value; + if (!parse_integer(max_color_value) || max_color_value != 255) { + return false; + } + + image.pixel_data = reinterpret_cast<u8 const*>(m_cursor); + + return true; +} + +bool BootPPMParser::check_position() const +{ + if (m_cursor >= m_buffer_end) { + return false; + } + return true; +} + +bool BootPPMParser::parse_magic() +{ + if (m_cursor[0] != 'P' || m_cursor[1] != '6') { + return false; + } + m_cursor += 2; + + return check_position(); +} + +bool BootPPMParser::parse_new_line() +{ + if (*m_cursor != '\n') { + return false; + } + ++m_cursor; + + return check_position(); +} + +bool BootPPMParser::parse_comment() +{ + if (*m_cursor == '#') { + // Skip to the next new line character + while (check_position() && *m_cursor != '\n') { + ++m_cursor; + } + ++m_cursor; + } + + return check_position(); +} + +bool BootPPMParser::parse_integer(u32& value) +{ + auto begin = m_cursor; + while (check_position() && *m_cursor != ' ' && *m_cursor != '\n') { + ++m_cursor; + } + auto end = m_cursor; + ++m_cursor; + + if (!check_position()) { + return false; + } + + value = 0; + u32 multiplier = 1; + while (--end >= begin) { + value += multiplier * (*end - '0'); + multiplier *= 10; + } + + return true; +} + +} diff --git a/Kernel/Arch/aarch64/BootPPMParser.h b/Kernel/Arch/aarch64/BootPPMParser.h new file mode 100644 index 0000000000..a311a50a7a --- /dev/null +++ b/Kernel/Arch/aarch64/BootPPMParser.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> + +namespace Prekernel { + +// Quick parser for .ppm image format (raw PortablePixMap) +// This is much simpler version than userland implementation in PPMLoader.cpp +class BootPPMParser { +public: + struct { + u32 width = 0; + u32 height = 0; + u8 const* pixel_data = nullptr; + } image; + + BootPPMParser(u8 const* buffer, u32 buffer_size); + + bool parse(); + +private: + char const* m_cursor; + char const* m_buffer_end; + + bool check_position() const; + bool parse_magic(); + bool parse_new_line(); + bool parse_comment(); + bool parse_integer(u32& value); +}; + +} diff --git a/Kernel/Arch/aarch64/Framebuffer.cpp b/Kernel/Arch/aarch64/Framebuffer.cpp new file mode 100644 index 0000000000..fb99de9adf --- /dev/null +++ b/Kernel/Arch/aarch64/Framebuffer.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/Framebuffer.h> +#include <Kernel/Arch/aarch64/FramebufferMailboxMessages.h> +#include <Kernel/Arch/aarch64/Utils.h> + +namespace Prekernel { + +Framebuffer::Framebuffer() +{ + // FIXME: query HDMI for best mode + // https://github.com/raspberrypi/userland/blob/master/host_applications/linux/apps/tvservice/tvservice.c + m_width = 1280; + m_height = 720; + m_depth = 32; + m_initialized = false; + + struct __attribute__((aligned(16))) { + Mailbox::MessageHeader header; + FramebufferSetPhysicalSizeMboxMessage set_physical_size; + FramebufferSetVirtualSizeMboxMessage set_virtual_size; + FramebufferSetVirtualOffsetMboxMessage set_virtual_offset; + FramebufferSetDepthMboxMessage set_depth; + FramebufferSetPixelOrderMboxMessage set_pixel_order; + FramebufferAllocateBufferMboxMessage allocate_buffer; + FramebufferGetPithMboxMessage get_pitch; + Mailbox::MessageTail tail; + } message_queue; + + message_queue.header.set_queue_size(sizeof(message_queue)); + message_queue.set_physical_size.width = m_width; + message_queue.set_physical_size.height = m_height; + message_queue.set_virtual_size.width = m_width; + message_queue.set_virtual_size.height = m_height; + + // FIXME! those next 2 lines crash... + // message_queue.set_virtual_offset.x = 0; + // message_queue.set_virtual_offset.y = 0; + + message_queue.set_depth.depth_bits = 32; + message_queue.set_pixel_order.pixel_order = FramebufferSetPixelOrderMboxMessage::PixelOrder::RGB; + message_queue.allocate_buffer.alignment = 4096; + + if (!Mailbox::the().send_queue(&message_queue, sizeof(message_queue))) { + warnln("Framebuffer(): Mailbox send failed."); + return; + } + + // Now message queue contains responses. Process them. + + if (message_queue.set_physical_size.width != m_width || message_queue.set_physical_size.height != m_height) { + warnln("Framebuffer(): Setting physical dimension failed."); + return; + } + + if (message_queue.set_virtual_size.width != m_width || message_queue.set_virtual_size.height != m_height) { + warnln("Framebuffer(): Setting virtual dimension failed."); + return; + } + + if (message_queue.set_virtual_offset.x != 0 || message_queue.set_virtual_offset.y != 0) { + warnln("Framebuffer(): Setting virtual offset failed."); + return; + } + + if (message_queue.set_depth.depth_bits != m_depth) { + warnln("Framebuffer(): Setting depth failed."); + return; + } + + if (message_queue.allocate_buffer.size == 0 || message_queue.allocate_buffer.address == 0) { + warnln("Framebuffer(): Allocating buffer failed."); + return; + } + + if (message_queue.get_pitch.pitch == 0) { + warnln("Framebuffer(): Retrieving pitch failed."); + return; + } + + // Convert GPU address space to RAM + // GPU maps memory from 0x80000000 instead of 0x00000000 + m_gpu_buffer = reinterpret_cast<u8*>(message_queue.allocate_buffer.address & 0x3FFFFFFF); + + m_buffer_size = message_queue.allocate_buffer.size; + m_pitch = message_queue.get_pitch.pitch; + + switch (message_queue.set_pixel_order.pixel_order) { + case FramebufferSetPixelOrderMboxMessage::PixelOrder::RGB: + m_pixel_order = PixelOrder::RGB; + break; + case FramebufferSetPixelOrderMboxMessage::PixelOrder::BGR: + m_pixel_order = PixelOrder::BGR; + break; + default: + warnln("Framebuffer(): Unsupported pixel order reported by GPU."); + m_pixel_order = PixelOrder::RGB; + break; + } + + dbgln("Initialized framebuffer: 1280 x 720 @ 32 bits"); + m_initialized = true; +} + +Framebuffer& Framebuffer::the() +{ + static Framebuffer instance; + return instance; +} +} diff --git a/Kernel/Arch/aarch64/Framebuffer.h b/Kernel/Arch/aarch64/Framebuffer.h new file mode 100644 index 0000000000..2cde37dfdb --- /dev/null +++ b/Kernel/Arch/aarch64/Framebuffer.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> + +namespace Prekernel { + +class Framebuffer { +public: + enum class PixelOrder { + RGB, + BGR, + }; + + static Framebuffer& the(); + + bool initialized() const { return m_initialized; } + u16 width() const { return m_width; } + u16 height() const { return m_height; } + u8 depth() const { return m_depth; } + u8* gpu_buffer() const { return m_gpu_buffer; } + u32 buffer_size() const { return m_buffer_size; } + u32 pitch() const { return m_pitch; } + PixelOrder pixel_order() { return m_pixel_order; } + +private: + u16 m_width; + u16 m_height; + u8 m_depth; + u8* m_gpu_buffer; + u32 m_buffer_size; + u32 m_pitch; + bool m_initialized; + PixelOrder m_pixel_order; + + Framebuffer(); +}; +} diff --git a/Kernel/Arch/aarch64/FramebufferMailboxMessages.h b/Kernel/Arch/aarch64/FramebufferMailboxMessages.h new file mode 100644 index 0000000000..7dab801b73 --- /dev/null +++ b/Kernel/Arch/aarch64/FramebufferMailboxMessages.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <Kernel/Arch/aarch64/Mailbox.h> + +namespace Prekernel { + +class FramebufferSetPhysicalSizeMboxMessage : public Mailbox::Message { +public: + u32 width; + u32 height; + + FramebufferSetPhysicalSizeMboxMessage() + : Mailbox::Message(0x48003, 8) + { + width = 0; + height = 0; + } +}; +static_assert(sizeof(FramebufferSetPhysicalSizeMboxMessage) == 20); + +class FramebufferSetVirtualSizeMboxMessage : public Mailbox::Message { +public: + u32 width; + u32 height; + + FramebufferSetVirtualSizeMboxMessage() + : Mailbox::Message(0x48004, 8) + { + width = 0; + height = 0; + } +}; +static_assert(sizeof(FramebufferSetVirtualSizeMboxMessage) == 20); + +class FramebufferSetVirtualOffsetMboxMessage : public Mailbox::Message { +public: + u32 x; + u32 y; + + FramebufferSetVirtualOffsetMboxMessage() + : Mailbox::Message(0x48009, 8) + { + x = 0; + y = 0; + } +}; +static_assert(sizeof(FramebufferSetVirtualOffsetMboxMessage) == 20); + +class FramebufferSetDepthMboxMessage : public Mailbox::Message { +public: + u32 depth_bits; + + FramebufferSetDepthMboxMessage() + : Mailbox::Message(0x48005, 4) + { + depth_bits = 0; + } +}; +static_assert(sizeof(FramebufferSetDepthMboxMessage) == 16); + +class FramebufferSetPixelOrderMboxMessage : public Mailbox::Message { +public: + enum PixelOrder : u32 { + BGR = 0, + RGB = 1 + }; + + PixelOrder pixel_order; + + FramebufferSetPixelOrderMboxMessage() + : Mailbox::Message(0x48006, 4) + { + pixel_order = PixelOrder::BGR; + } +}; +static_assert(sizeof(FramebufferSetPixelOrderMboxMessage) == 16); + +class FramebufferAllocateBufferMboxMessage : public Mailbox::Message { +public: + union { + u32 alignment; + u32 address; + }; + u32 size = 0; + + FramebufferAllocateBufferMboxMessage() + : Mailbox::Message(0x40001, 8) + { + alignment = 0; + size = 0; + } +}; +static_assert(sizeof(FramebufferAllocateBufferMboxMessage) == 20); + +class FramebufferGetPithMboxMessage : public Mailbox::Message { +public: + u32 pitch; + + FramebufferGetPithMboxMessage() + : Mailbox::Message(0x40008, 4) + { + pitch = 0; + } +}; +static_assert(sizeof(FramebufferGetPithMboxMessage) == 16); + +} diff --git a/Kernel/Arch/aarch64/GPIO.cpp b/Kernel/Arch/aarch64/GPIO.cpp new file mode 100644 index 0000000000..bffba6bade --- /dev/null +++ b/Kernel/Arch/aarch64/GPIO.cpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/GPIO.h> +#include <Kernel/Arch/aarch64/MMIO.h> + +extern "C" void wait_cycles(int n); + +namespace Prekernel { + +// See BCM2835-ARM-Peripherals.pdf section "6 General Purpose I/O" or bcm2711-peripherals.pdf "Chapter 5. General Purpose I/O". + +// "6.1 Register View" / "5.2 Register View" + +struct PinData { + u32 bits[2]; + u32 reserved; +}; + +struct GPIOControlRegisters { + u32 function_select[6]; // Every u32 stores a 3-bit function code for 10 pins. + u32 reserved; + PinData output_set; + PinData output_clear; + PinData level; + PinData event_detect_status; + PinData rising_edge_detect_enable; + PinData falling_edge_detect_enable; + PinData high_detect_enable; + PinData low_detect_enable; + PinData async_rising_edge_detect_enable; + PinData async_falling_edge_detect_enable; + u32 pull_up_down_enable; + PinData pull_up_down_enable_clock; + u32 test; +}; + +GPIO::GPIO() + : m_registers(MMIO::the().peripheral<GPIOControlRegisters>(0x20'0000)) +{ +} + +GPIO& GPIO::the() +{ + static GPIO instance; + return instance; +} + +void GPIO::set_pin_function(unsigned pin_number, PinFunction function) +{ + // pin_number must be <= 53. We can't VERIFY() that since this function runs too early to print assertion failures. + + unsigned function_select_index = pin_number / 10; + unsigned function_select_bits_start = (pin_number % 10) * 3; + + u32 function_bits = m_registers->function_select[function_select_index]; + function_bits = (function_bits & ~(0b111 << function_select_bits_start)) | (static_cast<u32>(function) << function_select_bits_start); + m_registers->function_select[function_select_index] = function_bits; +} + +void GPIO::internal_enable_pins(u32 enable[2], PullUpDownState state) +{ + // Section "GPIO Pull-up/down Clock Registers (GPPUDCLKn)": + // The GPIO Pull-up/down Clock Registers control the actuation of internal pull-downs on + // the respective GPIO pins. These registers must be used in conjunction with the GPPUD + // register to effect GPIO Pull-up/down changes. The following sequence of events is + // required: + // 1. Write to GPPUD to set the required control signal (i.e. Pull-up or Pull-Down or neither + // to remove the current Pull-up/down) + m_registers->pull_up_down_enable = static_cast<u32>(state); + + // 2. Wait 150 cycles – this provides the required set-up time for the control signal + wait_cycles(150); + + // 3. Write to GPPUDCLK0/1 to clock the control signal into the GPIO pads you wish to + // modify – NOTE only the pads which receive a clock will be modified, all others will + // retain their previous state. + m_registers->pull_up_down_enable_clock.bits[0] = enable[0]; + m_registers->pull_up_down_enable_clock.bits[1] = enable[1]; + + // 4. Wait 150 cycles – this provides the required hold time for the control signal + wait_cycles(150); + + // 5. Write to GPPUD to remove the control signal + m_registers->pull_up_down_enable = 0; + + // 6. Write to GPPUDCLK0/1 to remove the clock + m_registers->pull_up_down_enable_clock.bits[0] = 0; + m_registers->pull_up_down_enable_clock.bits[1] = 0; + + // bcm2711-peripherals.pdf documents GPIO_PUP_PDN_CNTRL_REG[4] registers that store 2 bits state per register, similar to function_select. + // I don't know if the RPi3 has that already, so this uses the old BCM2835 approach for now. +} + +} diff --git a/Kernel/Arch/aarch64/GPIO.h b/Kernel/Arch/aarch64/GPIO.h new file mode 100644 index 0000000000..4f38f9c376 --- /dev/null +++ b/Kernel/Arch/aarch64/GPIO.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Array.h> +#include <AK/Types.h> + +namespace Prekernel { + +struct GPIOControlRegisters; + +// Can configure the general-purpose I/O registers on a Raspberry Pi. +class GPIO { +public: + enum class PinFunction { + Input = 0, + Output = 1, + Alternate0 = 0b100, + Alternate1 = 0b101, + Alternate2 = 0b110, + Alternate3 = 0b111, + Alternate4 = 0b011, + Alternate5 = 0b010, + }; + + static GPIO& the(); + + void set_pin_function(unsigned pin_number, PinFunction); + + enum class PullUpDownState { + Disable = 0, + PullDown = 1, + PullUp = 2, + }; + + template<size_t N> + void set_pin_pull_up_down_state(Array<int, N> pins, PullUpDownState state) + { + u32 enable[2] = {}; + for (int pin : pins) { + if (pin < 32) + enable[0] |= (1 << pin); + else + enable[1] |= (1 << (pin - 32)); + } + internal_enable_pins(enable, state); + } + +private: + GPIO(); + void internal_enable_pins(u32 enable[2], PullUpDownState state); + + GPIOControlRegisters volatile* m_registers; +}; + +} diff --git a/Kernel/Arch/aarch64/MMIO.cpp b/Kernel/Arch/aarch64/MMIO.cpp new file mode 100644 index 0000000000..a6b270a17d --- /dev/null +++ b/Kernel/Arch/aarch64/MMIO.cpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/MMIO.h> +#include <Kernel/Arch/aarch64/MainIdRegister.h> + +namespace Prekernel { + +MMIO::MMIO() + : m_base_address(0xFE00'0000) +{ + MainIdRegister id; + if (id.part_num() <= MainIdRegister::RaspberryPi3) + m_base_address = 0x3F00'0000; +} + +MMIO& MMIO::the() +{ + static MMIO instance; + return instance; +} + +} diff --git a/Kernel/Arch/aarch64/MMIO.h b/Kernel/Arch/aarch64/MMIO.h new file mode 100644 index 0000000000..895319d869 --- /dev/null +++ b/Kernel/Arch/aarch64/MMIO.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> + +namespace Prekernel { + +// Knows about memory-mapped IO addresses on the Broadcom family of SOCs used in Raspberry Pis. +// RPi3 is the first Raspberry Pi that supports aarch64. +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf (RPi3) +// https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf (RPi4 Model B) +class MMIO { +public: + static MMIO& the(); + + u32 read(FlatPtr offset) { return *peripheral_address(offset); } + void write(FlatPtr offset, u32 value) { *peripheral_address(offset) = value; } + + u32 volatile* peripheral_address(FlatPtr offset) { return (u32 volatile*)(m_base_address + offset); } + template<class T> + T volatile* peripheral(FlatPtr offset) { return (T volatile*)peripheral_address(offset); } + + FlatPtr peripheral_base_address() const { return m_base_address; } + FlatPtr peripheral_end_address() const { return m_base_address + 0x00FFFFFF; } + +private: + MMIO(); + + unsigned int m_base_address; +}; + +} diff --git a/Kernel/Arch/aarch64/Mailbox.cpp b/Kernel/Arch/aarch64/Mailbox.cpp new file mode 100644 index 0000000000..d36a5305b4 --- /dev/null +++ b/Kernel/Arch/aarch64/Mailbox.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/MMIO.h> +#include <Kernel/Arch/aarch64/Mailbox.h> + +namespace Prekernel { + +// There's one mailbox at MBOX_BASE_OFFSET for reading responses from VideoCore, and one at MBOX_BASE_OFFSET + 0x20 for sending requests. +// Each has its own status word. + +constexpr u32 MBOX_BASE_OFFSET = 0xB880; +constexpr u32 MBOX_0 = MBOX_BASE_OFFSET; +constexpr u32 MBOX_1 = MBOX_BASE_OFFSET + 0x20; + +constexpr u32 MBOX_READ_DATA = MBOX_0; +constexpr u32 MBOX_READ_POLL = MBOX_0 + 0x10; +constexpr u32 MBOX_READ_SENDER = MBOX_0 + 0x14; +constexpr u32 MBOX_READ_STATUS = MBOX_0 + 0x18; +constexpr u32 MBOX_READ_CONFIG = MBOX_0 + 0x1C; + +constexpr u32 MBOX_WRITE_DATA = MBOX_1; +constexpr u32 MBOX_WRITE_STATUS = MBOX_1 + 0x18; + +constexpr u32 MBOX_RESPONSE_SUCCESS = 0x8000'0000; +constexpr u32 MBOX_RESPONSE_PARTIAL = 0x8000'0001; +constexpr u32 MBOX_REQUEST = 0; +constexpr u32 MBOX_FULL = 0x8000'0000; +constexpr u32 MBOX_EMPTY = 0x4000'0000; + +constexpr int ARM_TO_VIDEOCORE_CHANNEL = 8; + +Mailbox::Message::Message(u32 tag, u32 arguments_size) +{ + m_tag = tag; + m_arguments_size = arguments_size; + m_command_tag = MBOX_REQUEST; +} + +Mailbox::MessageHeader::MessageHeader() +{ + m_message_queue_size = 0; + m_command_tag = MBOX_REQUEST; +} + +bool Mailbox::MessageHeader::success() const +{ + return m_command_tag == MBOX_RESPONSE_SUCCESS; +} + +Mailbox& Mailbox::the() +{ + static Mailbox instance; + return instance; +} + +static void wait_until_we_can_write(MMIO& mmio) +{ + // Since nothing else writes to the mailbox, this wait is mostly cargo-culted. + // Most baremetal tutorials on the internet query MBOX_READ_STATUS here, which I think is incorrect and only works because this wait really isn't needed. + while (mmio.read(MBOX_WRITE_STATUS) & MBOX_FULL) + ; +} + +static void wait_for_reply(MMIO& mmio) +{ + while (mmio.read(MBOX_READ_STATUS) & MBOX_EMPTY) + ; +} + +bool Mailbox::send_queue(void* queue, u32 queue_size) const +{ + // According to Raspberry Pi specs this is the only channel implemented. + const u32 channel = ARM_TO_VIDEOCORE_CHANNEL; + + auto message_header = reinterpret_cast<MessageHeader*>(queue); + message_header->set_queue_size(queue_size); + + auto& mmio = MMIO::the(); + + // The mailbox interface has a FIFO for message delivery in both directions. + // Responses can be delivered out of order to requests, but we currently ever only send on request at once. + // It'd be nice to have an async interface here where we send a message, then return immediately, and read the response when an interrupt arrives. + // But for now, this is synchronous. + + wait_until_we_can_write(mmio); + + // The mailbox message is 32-bit based, so this assumes that message is in the first 4 GiB. + u32 request = static_cast<u32>(reinterpret_cast<FlatPtr>(queue) & ~0xF) | (channel & 0xF); + mmio.write(MBOX_WRITE_DATA, request); + + for (;;) { + wait_for_reply(mmio); + + u32 response = mmio.read(MBOX_READ_DATA); + // We keep at most one message in flight and do synchronous communication, so response will always be == request for us. + if (response == request) + return message_header->success(); + } + + return true; +} + +} diff --git a/Kernel/Arch/aarch64/Mailbox.h b/Kernel/Arch/aarch64/Mailbox.h new file mode 100644 index 0000000000..f54eea7c99 --- /dev/null +++ b/Kernel/Arch/aarch64/Mailbox.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> + +namespace Prekernel { + +// Can exchange mailbox messages with the Raspberry Pi's VideoCore chip. +// https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface +class Mailbox { +public: + // Base class for Mailbox messages. Implemented in subsystems that use Mailbox. + class Message { + protected: + Message(u32 tag, u32 arguments_size); + + private: + u32 m_tag; + u32 m_arguments_size; + u32 m_command_tag; + }; + + // Must be at the beginning of every command message queue + class MessageHeader { + public: + MessageHeader(); + + u32 queue_size() { return m_message_queue_size; } + void set_queue_size(u32 size) { m_message_queue_size = size; } + bool success() const; + + private: + u32 m_message_queue_size; + u32 m_command_tag; + }; + + // Must be at the end of every command message queue + class MessageTail { + private: + u32 m_empty_tag = 0; + }; + + static Mailbox& the(); + + // Sends message queue to VideoCore + bool send_queue(void* queue, u32 queue_size) const; +}; + +} diff --git a/Kernel/Arch/aarch64/MainIdRegister.cpp b/Kernel/Arch/aarch64/MainIdRegister.cpp new file mode 100644 index 0000000000..9b363a72b9 --- /dev/null +++ b/Kernel/Arch/aarch64/MainIdRegister.cpp @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/MainIdRegister.h> + +namespace Prekernel { + +MainIdRegister::MainIdRegister() +{ + unsigned int mrs; + asm volatile("mrs %x0, MIDR_EL1" + : "=r"(mrs)); + m_value = mrs; +} + +} diff --git a/Kernel/Arch/aarch64/MainIdRegister.h b/Kernel/Arch/aarch64/MainIdRegister.h new file mode 100644 index 0000000000..5f8d775acb --- /dev/null +++ b/Kernel/Arch/aarch64/MainIdRegister.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +namespace Prekernel { + +class MainIdRegister { +public: + MainIdRegister(); + + enum Implementer { + ArmLimited = 0x41, + }; + unsigned implementer() const { return (m_value >> 24) & 0xFF; } + unsigned variant() const { return (m_value >> 20) & 0xF; } + unsigned architecture() const { return (m_value >> 16) & 0xF; } + + enum PartNum { + RaspberryPi1 = 0xB76, + RaspberryPi2 = 0xC07, + RaspberryPi3 = 0xD03, + RaspberryPi4 = 0xD08, + }; + unsigned part_num() const { return (m_value >> 4) & 0xFFF; } + + unsigned revision() const { return m_value & 0xF; } + +private: + unsigned int m_value; +}; + +} diff --git a/Kernel/Arch/aarch64/Prekernel.h b/Kernel/Arch/aarch64/Prekernel.h new file mode 100644 index 0000000000..4bb284f7cd --- /dev/null +++ b/Kernel/Arch/aarch64/Prekernel.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2021, James Mintram <me@jamesrm.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +namespace Prekernel { + +void drop_to_exception_level_1(); +void init_prekernel_page_tables(); + +[[noreturn]] void panic(const char* msg); + +[[noreturn]] void halt(); + +} diff --git a/Kernel/Arch/aarch64/PrekernelCommon.cpp b/Kernel/Arch/aarch64/PrekernelCommon.cpp new file mode 100644 index 0000000000..5035ae5ab8 --- /dev/null +++ b/Kernel/Arch/aarch64/PrekernelCommon.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021, James Mintram <me@jamesrm.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/Prekernel.h> + +#include <Kernel/Arch/aarch64/ASM_wrapper.h> +#include <Kernel/Arch/aarch64/UART.h> + +namespace Prekernel { + +[[noreturn]] void panic(const char* msg) +{ + auto& uart = Prekernel::UART::the(); + + if (msg) { + uart.print_str(msg); + } + + Prekernel::halt(); +} + +[[noreturn]] void halt() +{ + for (;;) { + asm volatile("wfi"); + } +} + +} diff --git a/Kernel/Arch/aarch64/PrekernelExceptions.cpp b/Kernel/Arch/aarch64/PrekernelExceptions.cpp new file mode 100644 index 0000000000..9feb7fb821 --- /dev/null +++ b/Kernel/Arch/aarch64/PrekernelExceptions.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2021, James Mintram <me@jamesrm.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/ASM_wrapper.h> +#include <Kernel/Arch/aarch64/Aarch64_asm_utils.h> +#include <Kernel/Arch/aarch64/Prekernel.h> +#include <Kernel/Arch/aarch64/Registers.h> + +extern "C" void enter_el2_from_el3(); +extern "C" void enter_el1_from_el2(); + +using namespace Kernel; + +namespace Prekernel { + +static void drop_to_el2() +{ + Aarch64::SCR_EL3 secure_configuration_register_el3 = {}; + + secure_configuration_register_el3.ST = 1; // Don't trap access to Counter-timer Physical Secure registers + secure_configuration_register_el3.RW = 1; // Lower level to use Aarch64 + secure_configuration_register_el3.NS = 1; // Non-secure state + secure_configuration_register_el3.HCE = 1; // Enable Hypervisor instructions at all levels + + Aarch64::SCR_EL3::write(secure_configuration_register_el3); + + Aarch64::SPSR_EL3 saved_program_status_register_el3 = {}; + + // Mask (disable) all interrupts + saved_program_status_register_el3.A = 1; + saved_program_status_register_el3.I = 1; + saved_program_status_register_el3.F = 1; + saved_program_status_register_el3.D = 1; + + // Indicate EL1 as exception origin mode (so we go back there) + saved_program_status_register_el3.M = Aarch64::SPSR_EL3::Mode::EL2t; + + // Set the register + Aarch64::SPSR_EL3::write(saved_program_status_register_el3); + + // This will jump into os_start() below + enter_el2_from_el3(); +} +static void drop_to_el1() +{ + Aarch64::HCR_EL2 hypervisor_configuration_register_el2 = {}; + hypervisor_configuration_register_el2.RW = 1; // EL1 to use 64-bit mode + Aarch64::HCR_EL2::write(hypervisor_configuration_register_el2); + + Aarch64::SPSR_EL2 saved_program_status_register_el2 = {}; + + // Mask (disable) all interrupts + saved_program_status_register_el2.A = 1; + saved_program_status_register_el2.I = 1; + saved_program_status_register_el2.F = 1; + + // Indicate EL1 as exception origin mode (so we go back there) + saved_program_status_register_el2.M = Aarch64::SPSR_EL2::Mode::EL1t; + + Aarch64::SPSR_EL2::write(saved_program_status_register_el2); + enter_el1_from_el2(); +} + +static void set_up_el1() +{ + Aarch64::SCTLR_EL1 system_control_register_el1 = Aarch64::SCTLR_EL1::reset_value(); + + system_control_register_el1.UCT = 1; // Don't trap access to CTR_EL0 + system_control_register_el1.nTWE = 1; // Don't trap WFE instructions + system_control_register_el1.nTWI = 1; // Don't trap WFI instructions + system_control_register_el1.DZE = 1; // Don't trap DC ZVA instructions + system_control_register_el1.UMA = 1; // Don't trap access to DAIF (debugging) flags of EFLAGS register + system_control_register_el1.SA0 = 1; // Enable stack access alignment check for EL0 + system_control_register_el1.SA = 1; // Enable stack access alignment check for EL1 + system_control_register_el1.A = 1; // Enable memory access alignment check + + Aarch64::SCTLR_EL1::write(system_control_register_el1); +} + +void drop_to_exception_level_1() +{ + switch (Kernel::Aarch64::Asm::get_current_exception_level()) { + case Kernel::Aarch64::Asm::ExceptionLevel::EL3: + drop_to_el2(); + [[fallthrough]]; + case Kernel::Aarch64::Asm::ExceptionLevel::EL2: + drop_to_el1(); + [[fallthrough]]; + case Kernel::Aarch64::Asm::ExceptionLevel::EL1: + set_up_el1(); + break; + default: { + Prekernel::panic("FATAL: CPU booted in unsupported exception mode!\r\n"); + } + } +} + +} diff --git a/Kernel/Arch/aarch64/PrekernelMMU.cpp b/Kernel/Arch/aarch64/PrekernelMMU.cpp new file mode 100644 index 0000000000..38c953b600 --- /dev/null +++ b/Kernel/Arch/aarch64/PrekernelMMU.cpp @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2021, James Mintram <me@jamesrm.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Types.h> + +#include <Kernel/Arch/aarch64/Prekernel.h> + +#include <Kernel/Arch/aarch64/ASM_wrapper.h> +#include <Kernel/Arch/aarch64/MMIO.h> +#include <Kernel/Arch/aarch64/Registers.h> +#include <Kernel/Arch/aarch64/UART.h> + +// Documentation here for Aarch64 Address Translations +// https://documentation-service.arm.com/static/5efa1d23dbdee951c1ccdec5?token= + +using namespace Kernel; + +// These come from the linker script +extern u8 page_tables_phys_start[]; +extern u8 page_tables_phys_end[]; + +namespace Prekernel { + +// physical memory +constexpr u32 START_OF_NORMAL_MEMORY = 0x00000000; +constexpr u32 END_OF_NORMAL_MEMORY = 0x3EFFFFFF; + +// 4KiB page size was chosen for the prekernel to make this code slightly simpler +constexpr u32 GRANULE_SIZE = 0x1000; +constexpr u32 PAGE_TABLE_SIZE = 0x1000; + +// Documentation for translation table format +// https://developer.arm.com/documentation/101811/0101/Controlling-address-translation +constexpr u32 PAGE_DESCRIPTOR = 0b11; +constexpr u32 TABLE_DESCRIPTOR = 0b11; +constexpr u32 DESCRIPTOR_MASK = ~0b11; + +constexpr u32 ACCESS_FLAG = 1 << 10; + +// shareability +constexpr u32 OUTER_SHAREABLE = (2 << 8); +constexpr u32 INNER_SHAREABLE = (3 << 8); + +// these index into the MAIR attribute table +constexpr u32 NORMAL_MEMORY = (0 << 2); +constexpr u32 DEVICE_MEMORY = (1 << 2); + +ALWAYS_INLINE static u64* descriptor_to_pointer(FlatPtr descriptor) +{ + return (u64*)(descriptor & DESCRIPTOR_MASK); +} + +namespace { +class PageBumpAllocator { +public: + PageBumpAllocator(u64* start, u64* end) + : m_start(start) + , m_end(end) + , m_current(start) + { + if (m_start >= m_end) { + Prekernel::panic("Invalid memory range passed to PageBumpAllocator"); + } + if ((FlatPtr)m_start % PAGE_TABLE_SIZE != 0 || (FlatPtr)m_end % PAGE_TABLE_SIZE != 0) { + Prekernel::panic("Memory range passed into PageBumpAllocator not aligned to PAGE_TABLE_SIZE"); + } + } + + u64* take_page() + { + if (m_current == m_end) { + Prekernel::panic("Prekernel pagetable memory exhausted"); + } + + u64* page = m_current; + m_current += (PAGE_TABLE_SIZE / sizeof(FlatPtr)); + + zero_page(page); + return page; + } + +private: + void zero_page(u64* page) + { + // Memset all page table memory to zero + for (u64* p = page; p < page + (PAGE_TABLE_SIZE / sizeof(u64)); p++) { + *p = 0; + } + } + + const u64* m_start; + const u64* m_end; + u64* m_current; +}; +} + +static void insert_identity_entries_for_physical_memory_range(PageBumpAllocator& allocator, u64* page_table, FlatPtr start, FlatPtr end, u64 flags) +{ + // Not very efficient, but simple and it works. + for (FlatPtr addr = start; addr < end; addr += GRANULE_SIZE) { + // Each level has 9 bits (512 entries) + u64 level0_idx = (addr >> 39) & 0x1FF; + u64 level1_idx = (addr >> 30) & 0x1FF; + u64 level2_idx = (addr >> 21) & 0x1FF; + u64 level3_idx = (addr >> 12) & 0x1FF; + + u64* level1_table = page_table; + + if (level1_table[level0_idx] == 0) { + level1_table[level0_idx] = (FlatPtr)allocator.take_page(); + level1_table[level0_idx] |= TABLE_DESCRIPTOR; + } + + u64* level2_table = descriptor_to_pointer(level1_table[level0_idx]); + + if (level2_table[level1_idx] == 0) { + level2_table[level1_idx] = (FlatPtr)allocator.take_page(); + level2_table[level1_idx] |= TABLE_DESCRIPTOR; + } + + u64* level3_table = descriptor_to_pointer(level2_table[level1_idx]); + + if (level3_table[level2_idx] == 0) { + level3_table[level2_idx] = (FlatPtr)allocator.take_page(); + level3_table[level2_idx] |= TABLE_DESCRIPTOR; + } + + u64* level4_table = descriptor_to_pointer(level3_table[level2_idx]); + u64* l4_entry = &level4_table[level3_idx]; + *l4_entry = addr; + *l4_entry |= flags; + } +} + +static void build_identity_map(PageBumpAllocator& allocator) +{ + u64* level1_table = allocator.take_page(); + + u64 normal_memory_flags = ACCESS_FLAG | PAGE_DESCRIPTOR | INNER_SHAREABLE | NORMAL_MEMORY; + u64 device_memory_flags = ACCESS_FLAG | PAGE_DESCRIPTOR | OUTER_SHAREABLE | DEVICE_MEMORY; + + insert_identity_entries_for_physical_memory_range(allocator, level1_table, START_OF_NORMAL_MEMORY, END_OF_NORMAL_MEMORY, normal_memory_flags); + insert_identity_entries_for_physical_memory_range(allocator, level1_table, MMIO::the().peripheral_base_address(), MMIO::the().peripheral_end_address(), device_memory_flags); +} + +static void switch_to_page_table(u8* page_table) +{ + Aarch64::Asm::set_ttbr0_el1((FlatPtr)page_table); + Aarch64::Asm::set_ttbr1_el1((FlatPtr)page_table); +} + +static void activate_mmu() +{ + Aarch64::MAIR_EL1 mair_el1 = {}; + mair_el1.Attr[0] = 0xFF; // Normal memory + mair_el1.Attr[1] = 0b00000100; // Device-nGnRE memory (non-cacheble) + Aarch64::MAIR_EL1::write(mair_el1); + + // Configure cacheability attributes for memory associated with translation table walks + Aarch64::TCR_EL1 tcr_el1 = {}; + + tcr_el1.SH1 = Aarch64::TCR_EL1::InnerShareable; + tcr_el1.ORGN1 = Aarch64::TCR_EL1::NormalMemory_Outer_WriteBack_ReadAllocate_WriteAllocateCacheable; + tcr_el1.IRGN1 = Aarch64::TCR_EL1::NormalMemory_Inner_WriteBack_ReadAllocate_WriteAllocateCacheable; + + tcr_el1.SH0 = Aarch64::TCR_EL1::InnerShareable; + tcr_el1.ORGN0 = Aarch64::TCR_EL1::NormalMemory_Outer_WriteBack_ReadAllocate_WriteAllocateCacheable; + tcr_el1.IRGN0 = Aarch64::TCR_EL1::NormalMemory_Inner_WriteBack_ReadAllocate_WriteAllocateCacheable; + + tcr_el1.TG1 = Aarch64::TCR_EL1::TG1GranuleSize::Size_4KB; + tcr_el1.TG0 = Aarch64::TCR_EL1::TG0GranuleSize::Size_4KB; + + // Auto detect the Intermediate Physical Address Size + Aarch64::ID_AA64MMFR0_EL1 feature_register = Aarch64::ID_AA64MMFR0_EL1::read(); + tcr_el1.IPS = feature_register.PARange; + + Aarch64::TCR_EL1::write(tcr_el1); + + // Enable MMU in the system control register + Aarch64::SCTLR_EL1 sctlr_el1 = Aarch64::SCTLR_EL1::read(); + sctlr_el1.M = 1; //Enable MMU + Aarch64::SCTLR_EL1::write(sctlr_el1); + + Aarch64::Asm::flush(); +} + +void init_prekernel_page_tables() +{ + PageBumpAllocator allocator((u64*)page_tables_phys_start, (u64*)page_tables_phys_end); + build_identity_map(allocator); + switch_to_page_table(page_tables_phys_start); + activate_mmu(); +} + +} diff --git a/Kernel/Arch/aarch64/SerenityLogoRGB.ppm b/Kernel/Arch/aarch64/SerenityLogoRGB.ppm Binary files differnew file mode 100644 index 0000000000..6cf9490cd4 --- /dev/null +++ b/Kernel/Arch/aarch64/SerenityLogoRGB.ppm diff --git a/Kernel/Arch/aarch64/Timer.cpp b/Kernel/Arch/aarch64/Timer.cpp new file mode 100644 index 0000000000..0a703c180b --- /dev/null +++ b/Kernel/Arch/aarch64/Timer.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/MMIO.h> +#include <Kernel/Arch/aarch64/Mailbox.h> +#include <Kernel/Arch/aarch64/Timer.h> +#include <Kernel/Arch/aarch64/Utils.h> + +namespace Prekernel { + +// "12.1 System Timer Registers" / "10.2 System Timer Registers" +struct TimerRegisters { + u32 control_and_status; + u32 counter_low; + u32 counter_high; + u32 compare[4]; +}; + +// Bits of the `control_and_status` register. +// See "CS register" in Broadcom doc for details. +enum FlagBits { + SystemTimerMatch0 = 1 << 0, + SystemTimerMatch1 = 1 << 1, + SystemTimerMatch2 = 1 << 2, + SystemTimerMatch3 = 1 << 3, +}; + +Timer::Timer() + : m_registers(MMIO::the().peripheral<TimerRegisters>(0x3000)) +{ +} + +Timer& Timer::the() +{ + static Timer instance; + return instance; +} + +u64 Timer::microseconds_since_boot() +{ + u32 high = m_registers->counter_high; + u32 low = m_registers->counter_low; + if (high != m_registers->counter_high) { + high = m_registers->counter_high; + low = m_registers->counter_low; + } + return (static_cast<u64>(high) << 32) | low; +} + +class SetClockRateMboxMessage : Prekernel::Mailbox::Message { +public: + u32 clock_id; + u32 rate_hz; + u32 skip_setting_turbo; + + SetClockRateMboxMessage() + : Prekernel::Mailbox::Message(0x0003'8002, 12) + { + clock_id = 0; + rate_hz = 0; + skip_setting_turbo = 0; + } +}; + +u32 Timer::set_clock_rate(ClockID clock_id, u32 rate_hz, bool skip_setting_turbo) +{ + struct __attribute__((aligned(16))) { + Prekernel::Mailbox::MessageHeader header; + SetClockRateMboxMessage set_clock_rate; + Prekernel::Mailbox::MessageTail tail; + } message_queue; + + message_queue.set_clock_rate.clock_id = static_cast<u32>(clock_id); + message_queue.set_clock_rate.rate_hz = rate_hz; + message_queue.set_clock_rate.skip_setting_turbo = skip_setting_turbo ? 1 : 0; + + if (!Prekernel::Mailbox::the().send_queue(&message_queue, sizeof(message_queue))) { + warnln("Timer::set_clock_rate() failed!"); + return 0; + } + + return message_queue.set_clock_rate.rate_hz; +} + +} diff --git a/Kernel/Arch/aarch64/Timer.h b/Kernel/Arch/aarch64/Timer.h new file mode 100644 index 0000000000..27762930bf --- /dev/null +++ b/Kernel/Arch/aarch64/Timer.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> + +namespace Prekernel { + +struct TimerRegisters; + +class Timer { +public: + static Timer& the(); + + u64 microseconds_since_boot(); + + enum class ClockID { + Reserved = 0, + EMMC = 1, + UART = 2, + ARM = 3, + CORE = 4, + V3D = 5, + H264 = 6, + ISP = 7, + SDRAM = 8, + PIXEL = 9, + PWM = 10, + HEVC = 11, + EMMC2 = 12, + M2MC = 13, + PIXEL_BVB = 14, + }; + u32 set_clock_rate(ClockID, u32 rate_hz, bool skip_setting_turbo = true); + +private: + Timer(); + + TimerRegisters volatile* m_registers; +}; + +} diff --git a/Kernel/Arch/aarch64/UART.cpp b/Kernel/Arch/aarch64/UART.cpp new file mode 100644 index 0000000000..36852b6fce --- /dev/null +++ b/Kernel/Arch/aarch64/UART.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/GPIO.h> +#include <Kernel/Arch/aarch64/MMIO.h> +#include <Kernel/Arch/aarch64/Timer.h> +#include <Kernel/Arch/aarch64/UART.h> + +namespace Prekernel { + +// "13.4 Register View" / "11.5 Register View" +struct UARTRegisters { + u32 data; + u32 receive_status_or_error_clear; + u32 unused[4]; + u32 flag; + u32 unused2; + + u32 unused_ilpr; + u32 integer_baud_rate_divisor; // Only the lowest 16 bits are used. + u32 fractional_baud_rate_divisor; // Only the lowest 6 bits are used. + u32 line_control; + + u32 control; + u32 interrupt_fifo_level_select; + u32 interrupt_mask_set_clear; + u32 raw_interrupt_status; + + u32 masked_interrupt_status; + u32 interrupt_clear; + u32 dma_control; + u32 test_control; +}; + +// Bits of the `flag` register. +// See "FR register" in Broadcom doc for details. +enum FlagBits { + ClearToSend = 1 << 0, + UnsupportedDSR = 1 << 1, + UnsupportedDCD = 1 << 2, + UARTBusy = 1 << 3, + ReceiveFifoEmpty = 1 << 4, + TransmitFifoFull = 1 << 5, + ReceiveFifoFull = 1 << 6, + TransmitFifoEmpty = 1 << 7, +}; + +// Bits for the `line_control` register. +// See "LCRH register" in Broadcom doc for details. +enum LineControlBits { + SendBreak = 1 << 0, + EnableParityCheckingAndGeneration = 1 << 1, + EvenParity = 1 << 2, + TransmitTwoStopBits = 1 << 3, + EnableFIFOs = 1 << 4, + + WordLength5Bits = 0b00 << 5, + WordLength6Bits = 0b01 << 5, + WordLength7Bits = 0b10 << 5, + WordLength8Bits = 0b11 << 5, + + StickParity = 1 << 7, +}; + +// Bits for the `control` register. +// See "CR register" in Broadcom doc for details. From there: +// NOTE: Program the control registers as follows: +// 1. Disable the UART. +// 2. Wait for the end of transmission or reception of the current character. +// 3. Flush the transmit FIFO by setting the FEN bit to 0 in the Line Control Register, UART_LCRH. +// 4. Reprogram the Control Register, UART_CR. +// 5. Enable the UART +enum ControlBits { + UARTEnable = 1 << 0, + UnsupportedSIREN = 1 << 1, + UnsupportedSIRLP = 1 << 2, + // Bits 3-6 are reserved. + LoopbackEnable = 1 << 7, + TransmitEnable = 1 << 8, + ReceiveEnable = 1 << 9, + UnsupportedDTR = 1 << 10, + RequestToSend = 1 << 11, + UnsupportedOut1 = 1 << 12, + UnsupportedOut2 = 1 << 13, + RTSHardwareFlowControlEnable = 1 << 14, + CTSHardwareFlowControlEnable = 1 << 15, +}; + +UART::UART() + : m_registers(MMIO::the().peripheral<UARTRegisters>(0x20'1000)) +{ + // Disable UART while changing configuration. + m_registers->control = 0; + + // FIXME: Should wait for current transmission to end and should flush FIFO. + + constexpr int baud_rate = 115'200; + + // Set UART clock so that the baud rate divisor ends up as 1.0. + // FIXME: Not sure if this is a good UART clock rate. + u32 rate_in_hz = Timer::the().set_clock_rate(Timer::ClockID::UART, 16 * baud_rate); + + // The BCM's PL011 UART is alternate function 0 on pins 14 and 15. + auto& gpio = Prekernel::GPIO::the(); + gpio.set_pin_function(14, Prekernel::GPIO::PinFunction::Alternate0); + gpio.set_pin_function(15, Prekernel::GPIO::PinFunction::Alternate0); + gpio.set_pin_pull_up_down_state(Array { 14, 15 }, Prekernel::GPIO::PullUpDownState::Disable); + + // Clock and pins are configured. Turn UART on. + set_baud_rate(baud_rate, rate_in_hz); + m_registers->line_control = EnableFIFOs | WordLength8Bits; + + m_registers->control = UARTEnable | TransmitEnable | ReceiveEnable; +} + +UART& UART::the() +{ + static UART instance; + return instance; +} + +void UART::send(u32 c) +{ + wait_until_we_can_send(); + m_registers->data = c; +} + +u32 UART::receive() +{ + wait_until_we_can_receive(); + + // Mask out error bits. + return m_registers->data & 0xFF; +} + +void UART::set_baud_rate(int baud_rate, int uart_frequency_in_hz) +{ + // Broadcom doc: """Baud rate divisor BAUDDIV = (FUARTCLK/(16 * Baud rate))""". + // BAUDDIV is stored as a 16.6 fixed point value, so do computation scaled by (1 << 6) == 64. + // 64*(FUARTCLK/(16 * Baud rate)) == 4*FUARTCLK/(Baud rate). For rounding, add 0.5 == (Baud rate/2)/(Baud rate). + u32 baud_rate_divisor_fixed_point = (4 * uart_frequency_in_hz + baud_rate / 2) / baud_rate; + + m_registers->integer_baud_rate_divisor = baud_rate_divisor_fixed_point / 64; + m_registers->fractional_baud_rate_divisor = baud_rate_divisor_fixed_point % 64; +} + +void UART::wait_until_we_can_send() +{ + while (m_registers->flag & TransmitFifoFull) + ; +} + +void UART::wait_until_we_can_receive() +{ + while (m_registers->flag & ReceiveFifoEmpty) + ; +} + +} diff --git a/Kernel/Arch/aarch64/UART.h b/Kernel/Arch/aarch64/UART.h new file mode 100644 index 0000000000..79cef0a166 --- /dev/null +++ b/Kernel/Arch/aarch64/UART.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * Copyright (c) 2021, Jesse Buhagiar <jooster669@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include <AK/Types.h> + +namespace Prekernel { + +struct UARTRegisters; + +// Abstracts the PL011 UART on a Raspberry Pi. +// (The BCM2711 on a Raspberry Pi 4 has five PL011 UARTs; this is always the first of those.) +class UART { +public: + static UART& the(); + + void send(u32 c); + u32 receive(); + + void print_str(const char* s) + { + while (*s) + send(*s++); + } + void print_num(u64 n) + { + char buf[21]; + int i = 0; + do { + buf[i++] = (n % 10) + '0'; + n /= 10; + } while (n); + for (i--; i >= 0; i--) + send(buf[i]); + } + + void print_hex(u64 n) + { + char buf[17]; + static const char* digits = "0123456789ABCDEF"; + int i = 0; + do { + buf[i++] = digits[n % 16]; + n /= 16; + } while (n); + send(static_cast<u32>('0')); + send(static_cast<u32>('x')); + buf[16] = '\0'; + for (i--; i >= 0; i--) { + send(buf[i]); + } + } + +private: + UART(); + + void set_baud_rate(int baud_rate, int uart_frequency_in_hz); + void wait_until_we_can_send(); + void wait_until_we_can_receive(); + + UARTRegisters volatile* m_registers; +}; + +} diff --git a/Kernel/Arch/aarch64/Utils.cpp b/Kernel/Arch/aarch64/Utils.cpp new file mode 100644 index 0000000000..92d2f3a7d8 --- /dev/null +++ b/Kernel/Arch/aarch64/Utils.cpp @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <Kernel/Arch/aarch64/UART.h> +#include <Kernel/Arch/aarch64/Utils.h> + +void Prekernel::dbgln(const char* text) +{ + auto& uart = Prekernel::UART::the(); + uart.print_str(text); + uart.print_str("\r\n"); +} + +void Prekernel::warnln(const char* text) +{ + dbgln(text); +} diff --git a/Kernel/Arch/aarch64/Utils.h b/Kernel/Arch/aarch64/Utils.h new file mode 100644 index 0000000000..0c25610654 --- /dev/null +++ b/Kernel/Arch/aarch64/Utils.h @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +namespace Prekernel { + +// FIXME: to be replaced by real implementation from AK/Format.h +void dbgln(const char* text); +void warnln(const char* text); + +} diff --git a/Kernel/Arch/aarch64/boot.S b/Kernel/Arch/aarch64/boot.S new file mode 100644 index 0000000000..4c44b5288a --- /dev/null +++ b/Kernel/Arch/aarch64/boot.S @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +// In a specially-named text section so that the linker script can put it first in .text. +.section ".text.first" + +.global start +.type start, @function +start: + // Let only core 0 continue, put other cores to sleep. + mrs x13, MPIDR_EL1 + and x13, x13, 0xff + cbnz x13, _ZN9Prekernel4haltEv + + // Let stack start before .text for now. + // 512 kiB (0x80000) of stack are probably not sufficient, especially once we give the other cores some stack too, + // but for now it's ok. + msr SPSel, #0 //Use the same SP as we descend into EL1 + ldr x14, =start + mov sp, x14 + + // Clear BSS. + ldr x14, =start_of_bss + ldr x15, =size_of_bss_divided_by_8 +Lbss_clear_loop: + str xzr, [x14], #8 + subs x15, x15, #1 + bne Lbss_clear_loop + + b init diff --git a/Kernel/Arch/aarch64/init.cpp b/Kernel/Arch/aarch64/init.cpp new file mode 100644 index 0000000000..bce753da49 --- /dev/null +++ b/Kernel/Arch/aarch64/init.cpp @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2021, Nico Weber <thakis@chromium.org> + * Copyright (c) 2021, Marcin Undak <mcinek@gmail.com> + * Copyright (c) 2021, Jesse Buhagiar <jooster669@gmail.com> + * Copyright (c) 2022, the SerenityOS developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <AK/Types.h> + +#include <Kernel/Arch/aarch64/ASM_wrapper.h> +#include <Kernel/Arch/aarch64/Aarch64_asm_utils.h> +#include <Kernel/Arch/aarch64/BootPPMParser.h> +#include <Kernel/Arch/aarch64/Framebuffer.h> +#include <Kernel/Arch/aarch64/Mailbox.h> +#include <Kernel/Arch/aarch64/Prekernel.h> +#include <Kernel/Arch/aarch64/Timer.h> +#include <Kernel/Arch/aarch64/UART.h> +#include <Kernel/Arch/aarch64/Utils.h> + +static void draw_logo(); +static u32 query_firmware_version(); + +extern "C" void wait_cycles(int n); + +struct TrapFrame { + u64 x[31]; // Saved general purpose registers + u64 spsr_el1; // Save Processor Status Register, EL1 + u64 elr_el1; // Exception Link Reigster, EL1 + u64 tpidr_el1; // EL0 thread ID + u64 sp_el0; // EL0 stack pointer +}; + +extern "C" [[noreturn]] void halt(); +extern "C" [[noreturn]] void init(); +extern "C" void exception_common(TrapFrame const* const trap_frame); + +extern "C" [[noreturn]] void init() +{ + auto& uart = Prekernel::UART::the(); + + uart.print_str("\r\nWelcome to Serenity OS!\r\n"); + uart.print_str("Imagine this being your ideal operating system.\r\n"); + uart.print_str("Observed deviations from that ideal are shortcomings of your imagination.\r\n\r\n"); + + auto firmware_version = query_firmware_version(); + uart.print_str("Firmware version: "); + uart.print_num(firmware_version); + uart.print_str("\r\n"); + + uart.print_str("CPU started in: EL"); + uart.print_num(static_cast<u64>(Kernel::Aarch64::Asm::get_current_exception_level())); + uart.print_str("\r\n"); + + uart.print_str("Drop CPU to EL1\r\n"); + Prekernel::drop_to_exception_level_1(); + + // Load EL1 vector table + extern uintptr_t vector_table_el1; + el1_vector_table_install(&vector_table_el1); + + uart.print_str("Initialize MMU\r\n"); + Prekernel::init_prekernel_page_tables(); + + auto& framebuffer = Prekernel::Framebuffer::the(); + if (framebuffer.initialized()) { + draw_logo(); + } + + uart.print_str("Enter loop\r\n"); + + auto& timer = Prekernel::Timer::the(); + u64 start_musec = 0; + for (;;) { + u64 now_musec; + while ((now_musec = timer.microseconds_since_boot()) - start_musec < 1'000'000) + ; + start_musec = now_musec; + uart.print_str("Timer: "); + uart.print_num(now_musec); + uart.print_str("\r\n"); + } +} + +// FIXME: Share this with the Intel Prekernel. +extern size_t __stack_chk_guard; +size_t __stack_chk_guard; +extern "C" [[noreturn]] void __stack_chk_fail(); + +void __stack_chk_fail() +{ + Prekernel::halt(); +} + +[[noreturn]] void __assertion_failed(char const*, char const*, unsigned int, char const*) +{ + Prekernel::halt(); +} + +extern "C" void exception_common(TrapFrame const* const trap_frame) +{ + constexpr bool print_stack_frame = true; + + if constexpr (print_stack_frame) { + auto& uart = Prekernel::UART::the(); + + uart.print_str("Exception Generated by processor!\n"); + for (auto reg = 0; reg < 31; reg++) { + uart.print_str("x"); + uart.print_num(reg); + uart.print_str(": "); + uart.print_hex(trap_frame->x[reg]); + uart.print_str("\r\n"); + } + + // Special registers + uart.print_str("spsr_el1: "); + uart.print_hex(trap_frame->spsr_el1); + uart.print_str("\r\n"); + + uart.print_str("elr_el1: "); + uart.print_hex(trap_frame->elr_el1); + uart.print_str("\r\n"); + + uart.print_str("tpidr_el1: "); + uart.print_hex(trap_frame->tpidr_el1); + uart.print_str("\r\n"); + + uart.print_str("sp_el0: "); + uart.print_hex(trap_frame->sp_el0); + uart.print_str("\r\n"); + } +} + +class QueryFirmwareVersionMboxMessage : Prekernel::Mailbox::Message { +public: + u32 version; + + QueryFirmwareVersionMboxMessage() + : Prekernel::Mailbox::Message(0x0000'0001, 4) + { + version = 0; + } +}; + +static u32 query_firmware_version() +{ + struct __attribute__((aligned(16))) { + Prekernel::Mailbox::MessageHeader header; + QueryFirmwareVersionMboxMessage query_firmware_version; + Prekernel::Mailbox::MessageTail tail; + } message_queue; + + if (!Prekernel::Mailbox::the().send_queue(&message_queue, sizeof(message_queue))) { + return 0xffff'ffff; + } + + return message_queue.query_firmware_version.version; +} + +extern "C" const u32 serenity_boot_logo_start; +extern "C" const u32 serenity_boot_logo_size; + +static void draw_logo() +{ + Prekernel::BootPPMParser logo_parser(reinterpret_cast<const u8*>(&serenity_boot_logo_start), serenity_boot_logo_size); + if (!logo_parser.parse()) { + Prekernel::warnln("Invalid boot logo."); + return; + } + + auto& uart = Prekernel::UART::the(); + uart.print_str("Boot logo size: "); + uart.print_num(serenity_boot_logo_size); + uart.print_str("\r\n"); + uart.print_str("Width: "); + uart.print_num(logo_parser.image.width); + uart.print_str("\r\n"); + uart.print_str("Height: "); + uart.print_num(logo_parser.image.height); + uart.print_str("\r\n"); + + auto& framebuffer = Prekernel::Framebuffer::the(); + auto fb_ptr = framebuffer.gpu_buffer(); + auto image_left = (framebuffer.width() - logo_parser.image.width) / 2; + auto image_right = image_left + logo_parser.image.width; + auto image_top = (framebuffer.height() - logo_parser.image.height) / 2; + auto image_bottom = image_top + logo_parser.image.height; + auto logo_pixels = logo_parser.image.pixel_data; + + for (u32 y = 0; y < framebuffer.height(); y++) { + for (u32 x = 0; x < framebuffer.width(); x++) { + if (x >= image_left && x < image_right && y >= image_top && y < image_bottom) { + switch (framebuffer.pixel_order()) { + case Prekernel::Framebuffer::PixelOrder::RGB: + fb_ptr[0] = logo_pixels[0]; + fb_ptr[1] = logo_pixels[1]; + fb_ptr[2] = logo_pixels[2]; + break; + case Prekernel::Framebuffer::PixelOrder::BGR: + fb_ptr[0] = logo_pixels[2]; + fb_ptr[1] = logo_pixels[1]; + fb_ptr[2] = logo_pixels[0]; + break; + default: + Prekernel::warnln("Unsupported pixel format"); + Prekernel::halt(); + } + + logo_pixels += 3; + } else { + fb_ptr[0] = 0xBD; + fb_ptr[1] = 0xBD; + fb_ptr[2] = 0xBD; + } + + fb_ptr[3] = 0xFF; + fb_ptr += 4; + } + fb_ptr += framebuffer.pitch() - framebuffer.width() * 4; + } +} diff --git a/Kernel/Arch/aarch64/linker.ld b/Kernel/Arch/aarch64/linker.ld new file mode 100644 index 0000000000..e04e9290ab --- /dev/null +++ b/Kernel/Arch/aarch64/linker.ld @@ -0,0 +1,49 @@ +ENTRY(start) + +PHDRS +{ + text PT_LOAD ; + data PT_LOAD ; + bss PT_LOAD ; +} + +SECTIONS +{ + . = 0x00080000; + + .text ALIGN(4K) : AT (ADDR(.text)) + { + *(.text.first) + *(.text*) + } :text + + .rodata ALIGN(4K) : AT (ADDR(.rodata)) + { + *(.rodata*) + } :data + + .data ALIGN(4K) : AT (ADDR(.data)) + { + *(.data*) + } :data + + .bss ALIGN(4K) (NOLOAD) : AT (ADDR(.bss)) + { + start_of_bss = .; + *(.bss) + end_of_bss = .; + } :bss + + /* + FIXME: 8MB is enough space for all of the tables required to identity map + physical memory. 8M is wasteful, so this should be properly calculated. + */ + + . = ALIGN(4K); + page_tables_phys_start = .; + + . += 8M; + page_tables_phys_end = .; +} + +size_of_bss_divided_by_8 = (end_of_bss - start_of_bss) / 8; diff --git a/Kernel/Arch/aarch64/vector_table.S b/Kernel/Arch/aarch64/vector_table.S new file mode 100644 index 0000000000..826b38ba56 --- /dev/null +++ b/Kernel/Arch/aarch64/vector_table.S @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2021, Jesse Buhagiar <jooster669@gmail.com> + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +.section .text.vector_table + +#define TRAP_FRAME_SIZE 272 +#define SPSR_EL1_SLOT (31 * 8) +#define ELR_EL1_SLOT (32 * 8) +#define TPIDR_EL0_SLOT (33 * 8) +#define SP_EL0_SLOT (34 * 8) + +// Vector Table Entry macro. Each entry is aligned at 128 bytes, meaning we have +// at most that many instructions. +.macro table_entry label + .align 7 + b \label +.endm + +.macro unimplemented_entry + .align 7 + wfe + b . +.endm + +.extern exception_common + +// +// Save all register states to the current stack +// and enter the C++ exception handler +// +.macro save_current_context + // Allocate stack space for Trap Frame + sub sp, sp, #TRAP_FRAME_SIZE + + stp x0, x1, [sp, #(0 * 0)] + stp x2, x3, [sp, #(2 * 8)] + stp x4, x5, [sp, #(4 * 8)] + stp x6, x7, [sp, #(6 * 8)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(10 * 8)] + stp x12, x13, [sp, #(12 * 8)] + stp x14, x15, [sp, #(14 * 8)] + stp x16, x17, [sp, #(16 * 8)] + stp x18, x19, [sp, #(18 * 8)] + stp x20, x21, [sp, #(20 * 8)] + stp x22, x23, [sp, #(22 * 8)] + stp x24, x25, [sp, #(24 * 8)] + stp x26, x27, [sp, #(26 * 8)] + stp x28, x29, [sp, #(28 * 8)] + str x30, [sp, #(30 * 8)] + + // Let's save some special registers + mrs x0, spsr_el1 + str x0, [sp, #SPSR_EL1_SLOT] + mrs x0, elr_el1 + str x0, [sp, #ELR_EL1_SLOT] + mrs x0, tpidr_el0 + str x0, [sp, #TPIDR_EL0_SLOT] + mrs x0, sp_el0 + str x0, [sp, #SP_EL0_SLOT] + + // Move stack pointer into first argument register + // and jump to the C++ exception handler + mov x0, sp +.endm + +.macro restore_previous_context + // Restore special registers first + ldr x0, [sp, #SPSR_EL1_SLOT] + msr spsr_el1, x0 + ldr x0, [sp, #ELR_EL1_SLOT] + msr elr_el1, x0 + ldr x0, [sp, #TPIDR_EL0_SLOT] + msr tpidr_el0, x0 + ldr x0, [sp, #SP_EL0_SLOT] + msr sp_el0, x0 + + ldp x0, x1, [sp, #(0 * 0)] + ldp x2, x3, [sp, #(2 * 8)] + ldp x4, x5, [sp, #(4 * 8)] + ldp x6, x7, [sp, #(6 * 8)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(10 * 8)] + ldp x12, x13, [sp, #(12 * 8)] + ldp x14, x15, [sp, #(14 * 8)] + ldp x16, x17, [sp, #(16 * 8)] + ldp x18, x19, [sp, #(18 * 8)] + ldp x20, x21, [sp, #(20 * 8)] + ldp x22, x23, [sp, #(22 * 8)] + ldp x24, x25, [sp, #(24 * 8)] + ldp x26, x27, [sp, #(26 * 8)] + ldp x28, x29, [sp, #(28 * 8)] + ldr x30, [sp, #(30 * 8)] + + add sp, sp, #TRAP_FRAME_SIZE +.endm + +.global vector_table_el1 +.weak vector_table_el1 // Vector table is weak in case someone wants to hook us in C++ land :^) +.type vector_table_el1, @object + +// Vector table is 2KiB aligned (2^11) +.align 11 +vector_table_el1: + // Exceptions taken from Current EL, with SP_EL0 + unimplemented_entry + unimplemented_entry + unimplemented_entry + unimplemented_entry + + // Exceptions taken from Current EL, with SP_ELx, x>0 + table_entry synchronous_current_elsp_elx + table_entry irq_current_elsp_elx + table_entry fiq_current_elsp_elx + table_entry system_error_current_elsp_elx + + // Exceptions from Lower EL, where causing application is in AArch64 mode + unimplemented_entry + unimplemented_entry + unimplemented_entry + unimplemented_entry + + // Exceptions from Lower EL, where causing application is in AArch32 mode + unimplemented_entry + unimplemented_entry + unimplemented_entry + unimplemented_entry + +synchronous_current_elsp_elx: + save_current_context + bl exception_common + restore_previous_context + eret + +irq_current_elsp_elx: + save_current_context + bl exception_common + restore_previous_context + eret + +fiq_current_elsp_elx: + save_current_context + bl exception_common + restore_previous_context + eret + +system_error_current_elsp_elx: + save_current_context + bl exception_common + restore_previous_context + eret |