1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
/*
* Copyright (c) 2021, James Mintram <me@jamesrm.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Types.h>
#include <Kernel/Arch/aarch64/CPU.h>
#include <Kernel/Arch/PageDirectory.h>
#include <Kernel/Arch/aarch64/ASM_wrapper.h>
#include <Kernel/Arch/aarch64/RPi/MMIO.h>
#include <Kernel/Arch/aarch64/RPi/UART.h>
#include <Kernel/Arch/aarch64/Registers.h>
#include <Kernel/Panic.h>
// Documentation here for Aarch64 Address Translations
// https://documentation-service.arm.com/static/5efa1d23dbdee951c1ccdec5?token=
// These come from the linker script
extern u8 page_tables_phys_start[];
extern u8 page_tables_phys_end[];
namespace Kernel {
// physical memory
constexpr u32 START_OF_NORMAL_MEMORY = 0x00000000;
constexpr u32 END_OF_NORMAL_MEMORY = 0x3EFFFFFF;
ALWAYS_INLINE static u64* descriptor_to_pointer(FlatPtr descriptor)
{
return (u64*)(descriptor & DESCRIPTOR_MASK);
}
namespace {
class PageBumpAllocator {
public:
PageBumpAllocator(u64* start, u64* end)
: m_start(start)
, m_end(end)
, m_current(start)
{
if (m_start >= m_end) {
PANIC("Invalid memory range passed to PageBumpAllocator");
}
if ((FlatPtr)m_start % PAGE_TABLE_SIZE != 0 || (FlatPtr)m_end % PAGE_TABLE_SIZE != 0) {
PANIC("Memory range passed into PageBumpAllocator not aligned to PAGE_TABLE_SIZE");
}
}
u64* take_page()
{
if (m_current == m_end) {
PANIC("Prekernel pagetable memory exhausted");
}
u64* page = m_current;
m_current += (PAGE_TABLE_SIZE / sizeof(FlatPtr));
zero_page(page);
return page;
}
private:
void zero_page(u64* page)
{
// Memset all page table memory to zero
for (u64* p = page; p < page + (PAGE_TABLE_SIZE / sizeof(u64)); p++) {
*p = 0;
}
}
u64 const* m_start;
u64 const* m_end;
u64* m_current;
};
}
static void insert_identity_entries_for_physical_memory_range(PageBumpAllocator& allocator, u64* page_table, FlatPtr start, FlatPtr end, u64 flags)
{
// Not very efficient, but simple and it works.
for (FlatPtr addr = start; addr < end; addr += GRANULE_SIZE) {
// Each level has 9 bits (512 entries)
u64 level0_idx = (addr >> 39) & 0x1FF;
u64 level1_idx = (addr >> 30) & 0x1FF;
u64 level2_idx = (addr >> 21) & 0x1FF;
u64 level3_idx = (addr >> 12) & 0x1FF;
u64* level1_table = page_table;
if (level1_table[level0_idx] == 0) {
level1_table[level0_idx] = (FlatPtr)allocator.take_page();
level1_table[level0_idx] |= TABLE_DESCRIPTOR;
}
u64* level2_table = descriptor_to_pointer(level1_table[level0_idx]);
if (level2_table[level1_idx] == 0) {
level2_table[level1_idx] = (FlatPtr)allocator.take_page();
level2_table[level1_idx] |= TABLE_DESCRIPTOR;
}
u64* level3_table = descriptor_to_pointer(level2_table[level1_idx]);
if (level3_table[level2_idx] == 0) {
level3_table[level2_idx] = (FlatPtr)allocator.take_page();
level3_table[level2_idx] |= TABLE_DESCRIPTOR;
}
u64* level4_table = descriptor_to_pointer(level3_table[level2_idx]);
u64* l4_entry = &level4_table[level3_idx];
*l4_entry = addr;
*l4_entry |= flags;
}
}
static void build_identity_map(PageBumpAllocator& allocator)
{
u64* level1_table = allocator.take_page();
u64 normal_memory_flags = ACCESS_FLAG | PAGE_DESCRIPTOR | INNER_SHAREABLE | NORMAL_MEMORY;
u64 device_memory_flags = ACCESS_FLAG | PAGE_DESCRIPTOR | OUTER_SHAREABLE | DEVICE_MEMORY;
insert_identity_entries_for_physical_memory_range(allocator, level1_table, START_OF_NORMAL_MEMORY, END_OF_NORMAL_MEMORY, normal_memory_flags);
insert_identity_entries_for_physical_memory_range(allocator, level1_table, RPi::MMIO::the().peripheral_base_address(), RPi::MMIO::the().peripheral_end_address(), device_memory_flags);
}
static void switch_to_page_table(u8* page_table)
{
Aarch64::Asm::set_ttbr0_el1((FlatPtr)page_table);
Aarch64::Asm::set_ttbr1_el1((FlatPtr)page_table);
}
static void activate_mmu()
{
Aarch64::MAIR_EL1 mair_el1 = {};
mair_el1.Attr[0] = 0xFF; // Normal memory
mair_el1.Attr[1] = 0b00000100; // Device-nGnRE memory (non-cacheble)
Aarch64::MAIR_EL1::write(mair_el1);
// Configure cacheability attributes for memory associated with translation table walks
Aarch64::TCR_EL1 tcr_el1 = {};
tcr_el1.SH1 = Aarch64::TCR_EL1::InnerShareable;
tcr_el1.ORGN1 = Aarch64::TCR_EL1::NormalMemory_Outer_WriteBack_ReadAllocate_WriteAllocateCacheable;
tcr_el1.IRGN1 = Aarch64::TCR_EL1::NormalMemory_Inner_WriteBack_ReadAllocate_WriteAllocateCacheable;
tcr_el1.T1SZ = 16;
tcr_el1.SH0 = Aarch64::TCR_EL1::InnerShareable;
tcr_el1.ORGN0 = Aarch64::TCR_EL1::NormalMemory_Outer_WriteBack_ReadAllocate_WriteAllocateCacheable;
tcr_el1.IRGN0 = Aarch64::TCR_EL1::NormalMemory_Inner_WriteBack_ReadAllocate_WriteAllocateCacheable;
tcr_el1.T0SZ = 16;
tcr_el1.TG1 = Aarch64::TCR_EL1::TG1GranuleSize::Size_4KB;
tcr_el1.TG0 = Aarch64::TCR_EL1::TG0GranuleSize::Size_4KB;
// Auto detect the Intermediate Physical Address Size
Aarch64::ID_AA64MMFR0_EL1 feature_register = Aarch64::ID_AA64MMFR0_EL1::read();
tcr_el1.IPS = feature_register.PARange;
Aarch64::TCR_EL1::write(tcr_el1);
// Enable MMU in the system control register
Aarch64::SCTLR_EL1 sctlr_el1 = Aarch64::SCTLR_EL1::read();
sctlr_el1.M = 1; // Enable MMU
Aarch64::SCTLR_EL1::write(sctlr_el1);
Aarch64::Asm::flush();
}
void init_page_tables()
{
PageBumpAllocator allocator((u64*)page_tables_phys_start, (u64*)page_tables_phys_end);
build_identity_map(allocator);
switch_to_page_table(page_tables_phys_start);
activate_mmu();
}
}
|
