feat: aarch64 port - fix build system obstacles, add exception handling, page tables
This commit is contained in:
daniel committed
1 parent
aae4db01ad
commit
8b6379fb7e
19 files changed
+1207
-77
No files matched your search
+3
-1
@@ -291,7 +291,9 @@ cc-runtime-$(ARCH)/cc-runtime.a: GNUmakefile cc-runtime/*
|
||||
# so a clang-built non-x86 kernel is linked by ld.lld directly.
|
||||
ifneq ($(ARCH),x86_64)
|
||||
ifeq ($(CXX_IS_CLANG),1)
|
||||
KERNEL_LD ?= ld.lld
|
||||
# Without a system lld, fall back to the one rustup ships.
|
||||
KERNEL_LD ?= $(firstword $(shell command -v ld.lld) \
|
||||
$(wildcard $(HOME)/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld))
|
||||
override LD_EMULATION_aarch64 := aarch64elf
|
||||
override LD_EMULATION_riscv64 := elf64lriscv
|
||||
override LD_EMULATION_loongarch64 := elf64loongarch
|
||||
|
||||
@@ -12,4 +12,4 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define MONTAUK_BUILD_NUMBER 26
|
||||
#define MONTAUK_BUILD_NUMBER 29
|
||||
@@ -92,9 +92,10 @@ namespace Arch {
|
||||
asm volatile("dsb sy\n\tisb" ::: "memory");
|
||||
}
|
||||
|
||||
// The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0.
|
||||
inline void FlushTlbPage(uint64_t virtualAddress) {
|
||||
asm volatile("dsb ishst\n\ttlbi vaae1is, %0\n\tdsb ish\n\tisb"
|
||||
:: "r"(virtualAddress >> 12) : "memory");
|
||||
:: "r"((virtualAddress >> 12) & ((1ULL << 44) - 1)) : "memory");
|
||||
}
|
||||
|
||||
inline void FlushTlbAll() {
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* Exceptions.cpp
|
||||
* aarch64 synchronous exceptions: faults, user crashes, kernel panics
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#include <Arch/aarch64/InterruptFrame.hpp>
|
||||
#include <Arch/Cpu.hpp>
|
||||
#include <Arch/Smp.hpp>
|
||||
#include <Common/Panic.hpp>
|
||||
#include <Common/TerminationCodes.hpp>
|
||||
#include <CppLib/Stream.hpp>
|
||||
#include <Memory/Paging.hpp>
|
||||
#include <Sched/Scheduler.hpp>
|
||||
#include <Sched/CrashReport.hpp>
|
||||
#include <Api/Heap.hpp>
|
||||
#include <Timekeeping/Timer.hpp>
|
||||
|
||||
using namespace Kt;
|
||||
|
||||
namespace {
|
||||
|
||||
// ESR_EL1.EC exception classes
|
||||
constexpr uint64_t EcUnknown = 0x00;
|
||||
constexpr uint64_t EcIllegalState = 0x0E;
|
||||
constexpr uint64_t EcInstAbortLower = 0x20;
|
||||
constexpr uint64_t EcInstAbortSame = 0x21;
|
||||
constexpr uint64_t EcPcAlignment = 0x22;
|
||||
constexpr uint64_t EcDataAbortLower = 0x24;
|
||||
constexpr uint64_t EcDataAbortSame = 0x25;
|
||||
constexpr uint64_t EcSpAlignment = 0x26;
|
||||
constexpr uint64_t EcFpException = 0x2C;
|
||||
constexpr uint64_t EcBreakpointLower = 0x30;
|
||||
constexpr uint64_t EcBrk = 0x3C;
|
||||
|
||||
constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL;
|
||||
|
||||
uint64_t ExceptionClass(const Arch::PanicFrame* frame) {
|
||||
return (frame->esr >> 26) & 0x3F;
|
||||
}
|
||||
|
||||
bool IsAbort(uint64_t ec) {
|
||||
return ec == EcInstAbortLower || ec == EcInstAbortSame
|
||||
|| ec == EcDataAbortLower || ec == EcDataAbortSame;
|
||||
}
|
||||
|
||||
// Fault status code (ISS[5:0]) of an abort: translation faults mean
|
||||
// the page is not mapped, permission faults that it is.
|
||||
bool IsTranslationFault(uint64_t esr) {
|
||||
uint64_t fsc = esr & 0x3F;
|
||||
return fsc >= 0x04 && fsc <= 0x07;
|
||||
}
|
||||
|
||||
bool IsPermissionFault(uint64_t esr) {
|
||||
uint64_t fsc = esr & 0x3F;
|
||||
return fsc >= 0x0C && fsc <= 0x0F;
|
||||
}
|
||||
|
||||
const char* ExceptionName(uint64_t ec) {
|
||||
switch (ec) {
|
||||
case EcUnknown: return "Undefined Instruction";
|
||||
case EcIllegalState: return "Illegal Execution State";
|
||||
case EcInstAbortLower:
|
||||
case EcInstAbortSame: return "Instruction Abort";
|
||||
case EcPcAlignment: return "PC Alignment Fault";
|
||||
case EcDataAbortLower:
|
||||
case EcDataAbortSame: return "Data Abort";
|
||||
case EcSpAlignment: return "SP Alignment Fault";
|
||||
case EcFpException: return "Floating-Point Exception";
|
||||
case EcBrk: return "Breakpoint (BRK)";
|
||||
default:
|
||||
if (ec >= EcBreakpointLower && ec <= 0x35) return "Debug Exception";
|
||||
return "Synchronous Exception";
|
||||
}
|
||||
}
|
||||
|
||||
TerminationCodes TerminationFromClass(uint64_t ec) {
|
||||
if (ec == EcUnknown || ec == EcIllegalState) return TerminatedIllegalInstruction;
|
||||
if (ec == EcBrk || (ec >= EcBreakpointLower && ec <= 0x35)) return TerminatedBreakpoint;
|
||||
if (ec == EcFpException) return TerminatedArithmeticError;
|
||||
return TerminatedAccessViolation;
|
||||
}
|
||||
|
||||
template<size_t N>
|
||||
void CopyName(char (&dst)[N], const char* src) {
|
||||
size_t i = 0;
|
||||
for (; i + 1 < N && src[i]; i++) dst[i] = src[i];
|
||||
dst[i] = '\0';
|
||||
}
|
||||
|
||||
// Kill the faulting process with a crash report. Never returns.
|
||||
[[noreturn]] void TerminateUserProcess(Arch::PanicFrame* frame, uint64_t ec) {
|
||||
// Teardown can wait on devices and other CPUs; keep the clock running.
|
||||
Arch::EnableInterrupts();
|
||||
|
||||
auto* proc = Sched::GetCurrentProcessPtr();
|
||||
const char* name = ExceptionName(ec);
|
||||
KernelLogStream(ERROR, "Exception") << name << " in process \"" << proc->name
|
||||
<< "\" (pid " << proc->pid << ", ESR 0x" << base::hex << frame->esr
|
||||
<< ", PC 0x" << frame->elr << ") - process terminated";
|
||||
|
||||
CrashReport::Report rep{};
|
||||
rep.valid = true;
|
||||
rep.pid = proc->pid;
|
||||
CopyName(rep.processName, proc->name);
|
||||
rep.exceptionVector = (uint8_t)ec;
|
||||
CopyName(rep.exceptionName, name);
|
||||
rep.instructionPointer = frame->elr;
|
||||
rep.flags = frame->spsr;
|
||||
rep.stackPointer = frame->sp;
|
||||
|
||||
constexpr uint64_t snapshotBytes = CrashReport::StackSnapshotWords * sizeof(uint64_t);
|
||||
if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pageTableRoot, frame->sp,
|
||||
snapshotBytes, false)) {
|
||||
auto* userStack = (const volatile uint64_t*)frame->sp;
|
||||
for (int word = 0; word < CrashReport::StackSnapshotWords; word++)
|
||||
rep.stackWords[word] = userStack[word];
|
||||
rep.stackWordCount = CrashReport::StackSnapshotWords;
|
||||
}
|
||||
|
||||
if (IsAbort(ec)) {
|
||||
rep.faultingAddress = frame->far;
|
||||
rep.pfPresent = IsPermissionFault(frame->esr);
|
||||
rep.pfWrite = ec == EcDataAbortLower && (frame->esr & (1ULL << 6)) != 0;
|
||||
rep.pfUser = 1;
|
||||
rep.pfInstructionFetch = ec == EcInstAbortLower;
|
||||
}
|
||||
rep.timestampTick = Timekeeping::GetTicks();
|
||||
|
||||
CrashReport::AddReport(rep);
|
||||
Sched::SpawnCrashPad(proc->pid);
|
||||
|
||||
// As on x86: a sibling thread's fault must end the whole process.
|
||||
int code = TerminationFromClass(ec);
|
||||
if (Sched::GetCurrentThreadPtr() != proc) {
|
||||
Sched::KillProcess(proc->pid, code);
|
||||
} else {
|
||||
Sched::SetProcessExitCode(proc->pid, code);
|
||||
}
|
||||
|
||||
Sched::ExitProcess();
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
// A not-present user page: anonymous memory mapped lazily or the user
|
||||
// stack growing. Also reached from the kernel touching user buffers.
|
||||
bool TryResolveUserFault(const Arch::PanicFrame* frame, uint64_t ec) {
|
||||
if (!IsAbort(ec) || !IsTranslationFault(frame->esr)) return false;
|
||||
if (frame->far >= UserSpaceEnd) return false;
|
||||
|
||||
auto* cpu = Smp::TryGetCurrentCpuData();
|
||||
if (cpu == nullptr || cpu->currentSlot < 0 || Sched::GetCurrentPid() < 0) return false;
|
||||
|
||||
bool isData = ec == EcDataAbortLower || ec == EcDataAbortSame;
|
||||
Memory::VMM::PageFault fault{false, isData && (frame->esr & (1ULL << 6)) != 0, !isData};
|
||||
return montauk::abi::TryHandleAnonymousPageFault(frame->far, fault)
|
||||
|| Sched::TryGrowUserStack(frame->far);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
extern "C" void ArchSyncException(Arch::PanicFrame* frame, bool fromUser) {
|
||||
uint64_t ec = ExceptionClass(frame);
|
||||
|
||||
if (TryResolveUserFault(frame, ec)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fromUser && Sched::GetCurrentPid() >= 0) {
|
||||
TerminateUserProcess(frame, ec);
|
||||
}
|
||||
|
||||
Panic(ExceptionName(ec), frame);
|
||||
}
|
||||
|
||||
// FIQs are never routed, SErrors are asynchronous external aborts, and the
|
||||
// kernel neither runs on SP_EL0 nor supports AArch32 user code.
|
||||
extern "C" void ArchUnexpectedException(Arch::PanicFrame* frame, uint64_t kind) {
|
||||
static const char* const names[] = {
|
||||
"Exception on SP_EL0", "Unexpected FIQ", "SError (asynchronous abort)",
|
||||
"AArch32 exception",
|
||||
};
|
||||
Panic(kind < 4 ? names[kind] : "Unexpected exception", frame);
|
||||
}
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
namespace Arch {
|
||||
|
||||
// What the exception vectors save before calling into C++.
|
||||
// What the exception vectors (Vectors.S) save before calling into C++.
|
||||
struct PanicFrame {
|
||||
uint64_t x[31]; // x0-x30
|
||||
uint64_t sp; // interrupted SP (SP_EL0 for EL0, else SP_EL1)
|
||||
@@ -19,4 +19,6 @@ namespace Arch {
|
||||
uint64_t far; // FAR_EL1: faulting address
|
||||
};
|
||||
|
||||
static_assert(sizeof(PanicFrame) == 288, "must match Vectors.S");
|
||||
|
||||
}
|
||||
@@ -45,6 +45,17 @@ namespace Arch {
|
||||
|
||||
}
|
||||
|
||||
// IRQ exception vector (Vectors.S).
|
||||
// TODO(aarch64): acknowledge through the GIC CPU interface, translate the
|
||||
// INTID to a slot and call ArchIrqDispatch.
|
||||
extern "C" void ArchIrqException(void*, bool) {
|
||||
static bool reported = false;
|
||||
if (!reported) {
|
||||
reported = true;
|
||||
Kt::KernelLogStream(Kt::WARNING, "IRQ") << "aarch64: IRQ taken with no interrupt controller driver";
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ArchIrqDispatch(uint64_t irq, bool fromUser) {
|
||||
if (irq < Arch::IRQ_COUNT && Arch::g_irqHandlers[irq] != nullptr) {
|
||||
Arch::g_irqHandlers[irq]((uint8_t)irq, fromUser);
|
||||
|
||||
@@ -5,29 +5,67 @@
|
||||
*/
|
||||
|
||||
#include <Memory/Paging.hpp>
|
||||
#include <Memory/PageFrameAllocator.hpp>
|
||||
#include <Memory/HHDM.hpp>
|
||||
#include <Arch/Cpu.hpp>
|
||||
#include <Common/Panic.hpp>
|
||||
#include <CppLib/Spinlock.hpp>
|
||||
|
||||
// TODO(aarch64): 4 KiB-granule, 48-bit VA translation tables. The kernel
|
||||
// half lives in TTBR1_EL1 and is never switched; "address space roots"
|
||||
// handed to the generic kernel are TTBR0_EL1 tables, so
|
||||
// CreateUserAddressSpace has no kernel half to copy. MAIR_EL1 needs
|
||||
// Normal-WB, Normal-NC (MapWC) and Device-nGnRE (MapMMIO) indices.
|
||||
// 4 KiB granule, 48-bit virtual addresses, four levels (0-3). The kernel
|
||||
// half is translated through TTBR1_EL1 (g_kernelHalfRoot), which never
|
||||
// changes after Init. The "address space roots" the generic kernel switches
|
||||
// between are TTBR0_EL1 tables: a process's root holds only its own lower
|
||||
// half, and KernelRoot() is the TTBR0 table of kernel contexts, empty apart
|
||||
// from the EFI runtime identity mappings (so null pointers fault).
|
||||
//
|
||||
// The HHDM covers only what the memory map lists as RAM or firmware data:
|
||||
// unlike on x86, the holes between are device space on ARM machines, and a
|
||||
// cacheable mapping of device registers invites speculative accesses.
|
||||
|
||||
namespace Memory::VMM {
|
||||
|
||||
Paging* g_paging = nullptr;
|
||||
|
||||
static std::uint64_t g_hhdmLimit = 0;
|
||||
|
||||
static std::uint64_t g_kernelHalfRoot = 0;
|
||||
|
||||
std::uint64_t Paging::HhdmLimit() { return g_hhdmLimit; }
|
||||
|
||||
// Translation table descriptor bits (VMSAv8-64, stage 1)
|
||||
static constexpr uint64_t DescValid = 1ULL << 0;
|
||||
static constexpr uint64_t DescTable = 1ULL << 1; // table (levels 0-2) or page (level 3)
|
||||
static constexpr uint64_t DescAttrShift = 2; // AttrIndx[2:0], into MAIR_EL1
|
||||
static constexpr uint64_t DescAttrMask = 7ULL << DescAttrShift;
|
||||
static constexpr uint64_t DescApUser = 1ULL << 6; // AP[1]: EL0 may access
|
||||
static constexpr uint64_t DescApReadOnly = 1ULL << 7; // AP[2]
|
||||
static constexpr uint64_t DescShInner = 3ULL << 8;
|
||||
static constexpr uint64_t DescAf = 1ULL << 10; // access flag (no AF faults)
|
||||
static constexpr uint64_t DescNotGlobal = 1ULL << 11;
|
||||
static constexpr uint64_t DescPxn = 1ULL << 53;
|
||||
static constexpr uint64_t DescUxn = 1ULL << 54;
|
||||
static constexpr uint64_t DescAddrMask = 0x0000FFFFFFFFF000ULL;
|
||||
|
||||
// MAIR_EL1 attribute indices
|
||||
static constexpr uint64_t AttrNormal = 0; // write-back, read/write-allocate
|
||||
static constexpr uint64_t AttrNormalNc = 1; // non-cacheable (write-combining)
|
||||
static constexpr uint64_t AttrDevice = 2; // Device-nGnRE
|
||||
static constexpr uint64_t MairValue = 0xFFULL << (8 * AttrNormal)
|
||||
| 0x44ULL << (8 * AttrNormalNc)
|
||||
| 0x04ULL << (8 * AttrDevice);
|
||||
|
||||
static constexpr uint64_t KernelHalfBase = 0xFFFF000000000000ULL;
|
||||
static constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL;
|
||||
|
||||
// EFI memory types whose runtime regions are device registers
|
||||
static constexpr uint32_t EfiMemoryMappedIo = 11;
|
||||
static constexpr uint32_t EfiMemoryMappedIoPortSpace = 12;
|
||||
|
||||
static kcp::Mutex pagingLock;
|
||||
|
||||
void LockUserPaging() { pagingLock.Acquire(); }
|
||||
void UnlockUserPaging() { pagingLock.Release(); }
|
||||
|
||||
[[noreturn]] static void NotImplemented(const char* what) {
|
||||
Panic(what, nullptr);
|
||||
}
|
||||
|
||||
void ActivateKernelAddressSpace() {
|
||||
Arch::WritePageTableRoot(g_paging->KernelRoot());
|
||||
}
|
||||
@@ -36,75 +74,431 @@ namespace Memory::VMM {
|
||||
Arch::FlushTlbAll();
|
||||
}
|
||||
|
||||
using Descriptor = uint64_t;
|
||||
|
||||
static inline volatile Descriptor* TableAt(uint64_t phys) {
|
||||
return (volatile Descriptor*)Memory::HHDM(phys);
|
||||
}
|
||||
|
||||
static inline uint64_t Index(uint64_t va, int level) {
|
||||
return (va >> (39 - 9 * level)) & 0x1FF;
|
||||
}
|
||||
|
||||
// Order descriptor writes before the table walker (and later
|
||||
// instructions) can observe them.
|
||||
static inline void PublishTables() {
|
||||
asm volatile("dsb ishst\n\tisb" ::: "memory");
|
||||
}
|
||||
|
||||
static inline uint64_t ReadTtbr1() {
|
||||
uint64_t ttbr;
|
||||
asm volatile("mrs %0, ttbr1_el1" : "=r"(ttbr));
|
||||
return ttbr & Arch::TtbrAddressMask;
|
||||
}
|
||||
|
||||
// The level-3 descriptor for `va`, creating intermediate tables.
|
||||
// Returns nullptr when out of memory or when a block mapping covers
|
||||
// `va` (only the bootloader's tables contain blocks).
|
||||
static volatile Descriptor* WalkCreate(uint64_t root, uint64_t va) {
|
||||
uint64_t table = root;
|
||||
for (int level = 0; level < 3; level++) {
|
||||
volatile Descriptor* entry = &TableAt(table)[Index(va, level)];
|
||||
Descriptor desc = *entry;
|
||||
if ((desc & DescValid) == 0) {
|
||||
void* page = Memory::g_pfa->AllocateZeroed();
|
||||
if (page == nullptr) return nullptr;
|
||||
table = Memory::SubHHDM(page);
|
||||
// The zeroed table must be visible before the walker can
|
||||
// reach it.
|
||||
asm volatile("dsb ishst" ::: "memory");
|
||||
*entry = table | DescTable | DescValid;
|
||||
} else if ((desc & DescTable) == 0) {
|
||||
return nullptr;
|
||||
} else {
|
||||
table = desc & DescAddrMask;
|
||||
}
|
||||
}
|
||||
return &TableAt(table)[Index(va, 3)];
|
||||
}
|
||||
|
||||
// The level-3 descriptor for `va`, or nullptr if a table is missing.
|
||||
static volatile Descriptor* WalkExisting(uint64_t root, uint64_t va) {
|
||||
uint64_t table = root;
|
||||
for (int level = 0; level < 3; level++) {
|
||||
Descriptor desc = TableAt(table)[Index(va, level)];
|
||||
if ((desc & (DescValid | DescTable)) != (DescValid | DescTable)) return nullptr;
|
||||
table = desc & DescAddrMask;
|
||||
}
|
||||
return &TableAt(table)[Index(va, 3)];
|
||||
}
|
||||
|
||||
// Install a leaf. Replacing a live translation (other address or
|
||||
// attributes) needs break-before-make: invalidate, flush, then write.
|
||||
static void SetLeaf(volatile Descriptor* entry, Descriptor desc, uint64_t va) {
|
||||
Descriptor old = *entry;
|
||||
if (old == desc) return;
|
||||
if (old & DescValid) {
|
||||
*entry = 0;
|
||||
Arch::FlushTlbPage(va);
|
||||
}
|
||||
*entry = desc;
|
||||
PublishTables();
|
||||
}
|
||||
|
||||
static Descriptor KernelLeaf(uint64_t phys, uint64_t attr) {
|
||||
Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
|
||||
| (attr << DescAttrShift) | DescUxn;
|
||||
// Kernel memory stays executable, as on x86; device memory never is.
|
||||
if (attr != AttrNormal) desc |= DescPxn;
|
||||
return desc;
|
||||
}
|
||||
|
||||
static Descriptor UserLeaf(uint64_t phys, uint64_t attr, bool writable, bool executable) {
|
||||
Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
|
||||
| DescNotGlobal | (attr << DescAttrShift) | DescApUser | DescPxn;
|
||||
if (!writable) desc |= DescApReadOnly;
|
||||
if (!executable) desc |= DescUxn;
|
||||
return desc;
|
||||
}
|
||||
|
||||
static inline bool IsUserLeaf(Descriptor desc) {
|
||||
return (desc & (DescValid | DescTable)) == (DescValid | DescTable) && (desc & DescApUser);
|
||||
}
|
||||
|
||||
// Translate through a table tree that may contain block mappings (the
|
||||
// bootloader's). Returns 0 if unmapped.
|
||||
static uint64_t Translate(uint64_t root, uint64_t va) {
|
||||
uint64_t table = root;
|
||||
for (int level = 0; level <= 3; level++) {
|
||||
Descriptor desc = TableAt(table)[Index(va, level)];
|
||||
if ((desc & DescValid) == 0) return 0;
|
||||
|
||||
uint64_t span = 1ULL << (39 - 9 * level);
|
||||
bool leaf = level == 3 || (desc & DescTable) == 0;
|
||||
if (leaf) {
|
||||
if (level == 0) return 0; // no level-0 blocks with 4 KiB granules
|
||||
return ((desc & DescAddrMask) & ~(span - 1)) | (va & (span - 1));
|
||||
}
|
||||
table = desc & DescAddrMask;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void Paging::MapEfiRuntime(const montauk::boot::EfiInfo& efi) {
|
||||
if (efi.memoryMap == nullptr || efi.descriptorSize == 0) return;
|
||||
|
||||
struct EfiMemDesc {
|
||||
uint32_t Type;
|
||||
uint64_t PhysicalStart;
|
||||
uint64_t VirtualStart;
|
||||
uint64_t NumberOfPages;
|
||||
uint64_t Attribute;
|
||||
};
|
||||
|
||||
static constexpr uint64_t EFI_MEMORY_RUNTIME = 0x8000000000000000ULL;
|
||||
|
||||
auto* base = (uint8_t*)efi.memoryMap;
|
||||
uint64_t count = efi.memoryMapSize / efi.descriptorSize;
|
||||
|
||||
for (uint64_t i = 0; i < count; i++) {
|
||||
auto* desc = (EfiMemDesc*)(base + i * efi.descriptorSize);
|
||||
if ((desc->Attribute & EFI_MEMORY_RUNTIME) == 0) continue;
|
||||
|
||||
bool device = desc->Type == EfiMemoryMappedIo || desc->Type == EfiMemoryMappedIoPortSpace;
|
||||
for (uint64_t p = 0; p < desc->NumberOfPages; p++) {
|
||||
uint64_t addr = desc->PhysicalStart + p * 0x1000;
|
||||
// Firmware code runs at its HHDM alias (Efi::Init calls the
|
||||
// services there) and reaches its data at physical addresses.
|
||||
// Runtime regions are reserved memory, outside the HHDM.
|
||||
if (device) {
|
||||
MapMMIO(addr, addr);
|
||||
MapMMIO(addr, Memory::HHDM(addr));
|
||||
} else {
|
||||
Map(addr, addr);
|
||||
Map(addr, Memory::HHDM(addr));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FlushTLB();
|
||||
}
|
||||
|
||||
Paging::Paging() {
|
||||
NotImplemented("aarch64: virtual memory not implemented");
|
||||
rootPhys = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
|
||||
g_kernelHalfRoot = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
|
||||
}
|
||||
|
||||
void Paging::Init(std::uint64_t, std::uint64_t, const montauk::boot::MemoryMap&,
|
||||
const montauk::boot::Framebuffer&) {
|
||||
NotImplemented("aarch64: Paging::Init not implemented");
|
||||
// Kernel mappings of lower-half addresses (EFI identity mappings) go to
|
||||
// the kernel contexts' TTBR0 table.
|
||||
static uint64_t KernelRootFor(uint64_t kernelContextRoot, uint64_t va) {
|
||||
return va >= KernelHalfBase ? g_kernelHalfRoot : kernelContextRoot;
|
||||
}
|
||||
|
||||
std::uint64_t Paging::HhdmLimit() { return 0; }
|
||||
void Paging::Init(std::uint64_t kernelBaseVirt, std::uint64_t kernelSize, const montauk::boot::MemoryMap& memMap,
|
||||
const montauk::boot::Framebuffer& framebuffer) {
|
||||
if (Memory::HHDMBase < KernelHalfBase) {
|
||||
Panic("aarch64: HHDM outside the 48-bit kernel half", nullptr);
|
||||
}
|
||||
|
||||
void Paging::Map(std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::Map not implemented");
|
||||
uint64_t bootTtbr1 = ReadTtbr1();
|
||||
uint64_t bootTcr, bootMair;
|
||||
asm volatile("mrs %0, tcr_el1" : "=r"(bootTcr));
|
||||
asm volatile("mrs %0, mair_el1" : "=r"(bootMair));
|
||||
Kt::KernelLogStream(Kt::DEBUG, "VMM") << "Bootloader TCR 0x" << base::hex << bootTcr
|
||||
<< ", MAIR 0x" << bootMair << ", TTBR1 0x" << bootTtbr1;
|
||||
|
||||
// The kernel image, at the physical pages the bootloader loaded it to.
|
||||
for (uint64_t va = kernelBaseVirt; va < kernelBaseVirt + kernelSize; va += 0x1000) {
|
||||
uint64_t phys = Translate(bootTtbr1, va);
|
||||
if (phys == 0) Panic("aarch64: kernel page missing from the bootloader's tables", nullptr);
|
||||
Map(phys, va);
|
||||
}
|
||||
|
||||
// HHDM: every region that is real backing store (see the top of
|
||||
// this file). The limit is published for the AML interpreter.
|
||||
uint64_t maxPhysAddr = 0;
|
||||
for (size_t i = 0; i < memMap.count; i++) {
|
||||
const auto& region = memMap.regions[i];
|
||||
if (region.kind == montauk::boot::MemoryKind::Reserved
|
||||
|| region.kind == montauk::boot::MemoryKind::Unknown) {
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t start = region.base & ~0xFFFULL;
|
||||
uint64_t end = (region.base + region.length + 0xFFF) & ~0xFFFULL;
|
||||
bool framebufferRegion = region.kind == montauk::boot::MemoryKind::Framebuffer;
|
||||
for (uint64_t phys = start; phys < end; phys += 0x1000) {
|
||||
if (framebufferRegion) MapWC(phys, Memory::HHDM(phys));
|
||||
else Map(phys, Memory::HHDM(phys));
|
||||
}
|
||||
if (end > maxPhysAddr) maxPhysAddr = end;
|
||||
}
|
||||
g_hhdmLimit = maxPhysAddr;
|
||||
|
||||
// The framebuffer may not be listed in the memory map at all.
|
||||
if (framebuffer.address != 0) {
|
||||
uint64_t fbPhys = Memory::SubHHDM(framebuffer.address) & ~0xFFFULL;
|
||||
uint64_t fbPages = (framebuffer.height * framebuffer.pitch + 0xFFF) / 0x1000;
|
||||
for (uint64_t p = 0; p < fbPages; p++) {
|
||||
uint64_t phys = fbPhys + p * 0x1000;
|
||||
MapWC(phys, Memory::HHDM(phys));
|
||||
}
|
||||
}
|
||||
|
||||
// 48-bit VAs in both halves, 4 KiB granules, write-back walks of
|
||||
// inner-shareable tables; the physical size is what the CPU has.
|
||||
uint64_t mmfr0;
|
||||
asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0));
|
||||
uint64_t ips = mmfr0 & 0xF;
|
||||
if (ips > 5) ips = 5; // 52-bit output needs LPA2 descriptors
|
||||
uint64_t tcr = 16ULL // T0SZ
|
||||
| 1ULL << 8 | 1ULL << 10 // IRGN0, ORGN0: write-back
|
||||
| 3ULL << 12 // SH0: inner shareable
|
||||
| 0ULL << 14 // TG0: 4 KiB
|
||||
| 16ULL << 16 // T1SZ
|
||||
| 1ULL << 24 | 1ULL << 26 // IRGN1, ORGN1
|
||||
| 3ULL << 28 // SH1
|
||||
| 2ULL << 30 // TG1: 4 KiB
|
||||
| ips << 32; // IPS
|
||||
|
||||
// The new tables map the code running here at the same address, so
|
||||
// the switch is a register write followed by a full invalidate.
|
||||
asm volatile(
|
||||
"dsb ish\n\t"
|
||||
"msr mair_el1, %0\n\t"
|
||||
"msr tcr_el1, %1\n\t"
|
||||
"isb\n\t"
|
||||
"msr ttbr1_el1, %2\n\t"
|
||||
"msr ttbr0_el1, %3\n\t"
|
||||
"isb\n\t"
|
||||
"tlbi vmalle1is\n\t"
|
||||
"dsb ish\n\t"
|
||||
"isb"
|
||||
:: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(rootPhys) : "memory");
|
||||
|
||||
Kt::KernelLogStream(Kt::OK, "VMM") << "Switched to kernel translation tables (HHDM limit 0x"
|
||||
<< base::hex << g_hhdmLimit << ")";
|
||||
}
|
||||
|
||||
void Paging::MapMMIO(std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::MapMMIO not implemented");
|
||||
static void MapKernelPage(uint64_t root, uint64_t phys, uint64_t va, uint64_t attr, const char* what) {
|
||||
if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
|
||||
Panic(what, nullptr);
|
||||
}
|
||||
volatile Descriptor* entry = WalkCreate(KernelRootFor(root, va), va);
|
||||
if (entry == nullptr) {
|
||||
Panic("OOM in Paging (kernel page table allocation)", nullptr);
|
||||
}
|
||||
SetLeaf(entry, KernelLeaf(phys, attr), va);
|
||||
}
|
||||
|
||||
void Paging::MapWC(std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::MapWC not implemented");
|
||||
void Paging::Map(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||
MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormal,
|
||||
"Value that isn't page-aligned passed as address to Paging::Map!");
|
||||
}
|
||||
|
||||
void Paging::MapUser(std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::MapUser not implemented");
|
||||
void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||
MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrDevice,
|
||||
"Value that isn't page-aligned passed as address to Paging::MapMMIO!");
|
||||
}
|
||||
|
||||
std::uint64_t Paging::GetPhysAddr(std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::GetPhysAddr not implemented");
|
||||
void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||
MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormalNc,
|
||||
"Value that isn't page-aligned passed as address to Paging::MapWC!");
|
||||
}
|
||||
|
||||
void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||
if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
|
||||
Panic("Value that isn't page-aligned passed as address to Paging::MapUser!", nullptr);
|
||||
}
|
||||
volatile Descriptor* entry = WalkCreate(rootPhys, virtualAddress);
|
||||
if (entry == nullptr) {
|
||||
Panic("OOM in Paging::MapUser (page table allocation)", nullptr);
|
||||
}
|
||||
SetLeaf(entry, UserLeaf(physicalAddress, AttrNormal, true, true), virtualAddress);
|
||||
}
|
||||
|
||||
std::uint64_t Paging::GetPhysAddr(std::uint64_t root, std::uint64_t virtualAddress) {
|
||||
if (virtualAddress >= KernelHalfBase) {
|
||||
root = g_kernelHalfRoot;
|
||||
}
|
||||
return Translate(root, virtualAddress);
|
||||
}
|
||||
|
||||
std::uint64_t Paging::GetPhysAddr(std::uint64_t virtualAddress) {
|
||||
return GetPhysAddr(rootPhys, virtualAddress);
|
||||
}
|
||||
|
||||
// The kernel half lives in TTBR1, so a new address space starts empty.
|
||||
std::uint64_t Paging::CreateUserAddressSpace() {
|
||||
NotImplemented("aarch64: Paging::CreateUserAddressSpace not implemented");
|
||||
void* page = Memory::g_pfa->AllocateZeroed();
|
||||
if (page == nullptr) return 0;
|
||||
return Memory::SubHHDM(page);
|
||||
}
|
||||
|
||||
bool Paging::MapUserIn(std::uint64_t, std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::MapUserIn not implemented");
|
||||
bool Paging::MapUserIn(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||
return MapUserInPermissions(root, physicalAddress, virtualAddress, true, true);
|
||||
}
|
||||
|
||||
bool Paging::MapUserInPermissions(std::uint64_t, std::uint64_t, std::uint64_t, bool, bool) {
|
||||
NotImplemented("aarch64: Paging::MapUserInPermissions not implemented");
|
||||
static bool MapUserLeaf(uint64_t root, uint64_t phys, uint64_t va, Descriptor desc, const char* what) {
|
||||
pagingLock.Acquire();
|
||||
if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
|
||||
pagingLock.Release();
|
||||
Panic(what, nullptr);
|
||||
}
|
||||
volatile Descriptor* entry = WalkCreate(root, va);
|
||||
if (entry == nullptr) {
|
||||
pagingLock.Release();
|
||||
return false;
|
||||
}
|
||||
SetLeaf(entry, desc, va);
|
||||
pagingLock.Release();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Paging::ProtectUserIn(std::uint64_t, std::uint64_t, bool, bool) {
|
||||
NotImplemented("aarch64: Paging::ProtectUserIn not implemented");
|
||||
bool Paging::MapUserInPermissions(std::uint64_t root, std::uint64_t physicalAddress,
|
||||
std::uint64_t virtualAddress, bool writable, bool executable) {
|
||||
return MapUserLeaf(root, physicalAddress, virtualAddress,
|
||||
UserLeaf(physicalAddress, AttrNormal, writable, executable),
|
||||
"Non-aligned address in Paging::MapUserIn!");
|
||||
}
|
||||
|
||||
bool Paging::MapUserInWC(std::uint64_t, std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::MapUserInWC not implemented");
|
||||
bool Paging::MapUserInWC(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
|
||||
return MapUserLeaf(root, physicalAddress, virtualAddress,
|
||||
UserLeaf(physicalAddress, AttrNormalNc, true, false),
|
||||
"Non-aligned address in Paging::MapUserInWC!");
|
||||
}
|
||||
|
||||
void Paging::UnmapUserIn(std::uint64_t, std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::UnmapUserIn not implemented");
|
||||
// Permission changes need no break-before-make, only the TLB flush.
|
||||
bool Paging::ProtectUserIn(std::uint64_t root, std::uint64_t virtualAddress,
|
||||
bool writable, bool executable) {
|
||||
if ((virtualAddress & 0xFFFULL) != 0) return false;
|
||||
pagingLock.Acquire();
|
||||
volatile Descriptor* entry = WalkExisting(root, virtualAddress);
|
||||
if (entry == nullptr || !IsUserLeaf(*entry)) {
|
||||
pagingLock.Release();
|
||||
return false;
|
||||
}
|
||||
|
||||
Descriptor desc = *entry;
|
||||
if (writable) desc &= ~DescApReadOnly;
|
||||
else desc |= DescApReadOnly;
|
||||
if (executable) desc &= ~DescUxn;
|
||||
else desc |= DescUxn;
|
||||
*entry = desc;
|
||||
Arch::FlushTlbPage(virtualAddress);
|
||||
pagingLock.Release();
|
||||
return true;
|
||||
}
|
||||
|
||||
void Paging::FreeUserHalf(std::uint64_t) {
|
||||
NotImplemented("aarch64: Paging::FreeUserHalf not implemented");
|
||||
void Paging::UnmapUserIn(std::uint64_t root, std::uint64_t virtualAddress) {
|
||||
pagingLock.Acquire();
|
||||
volatile Descriptor* entry = WalkExisting(root, virtualAddress);
|
||||
if (entry != nullptr && (*entry & DescValid)) {
|
||||
*entry = 0;
|
||||
Arch::FlushTlbPage(virtualAddress);
|
||||
}
|
||||
pagingLock.Release();
|
||||
}
|
||||
|
||||
bool Paging::IsUserRangeAccessible(std::uint64_t, std::uint64_t, std::uint64_t, bool) {
|
||||
NotImplemented("aarch64: Paging::IsUserRangeAccessible not implemented");
|
||||
void Paging::FreeUserHalf(std::uint64_t root) {
|
||||
// User space is the lower 128 TiB: level-0 entries 0-255.
|
||||
for (int i0 = 0; i0 < 256; i0++) {
|
||||
Descriptor d0 = TableAt(root)[i0];
|
||||
if ((d0 & DescValid) == 0) continue;
|
||||
uint64_t l1 = d0 & DescAddrMask;
|
||||
|
||||
for (int i1 = 0; i1 < 512; i1++) {
|
||||
Descriptor d1 = TableAt(l1)[i1];
|
||||
if ((d1 & DescValid) == 0) continue;
|
||||
uint64_t l2 = d1 & DescAddrMask;
|
||||
|
||||
for (int i2 = 0; i2 < 512; i2++) {
|
||||
Descriptor d2 = TableAt(l2)[i2];
|
||||
if ((d2 & DescValid) == 0) continue;
|
||||
uint64_t l3 = d2 & DescAddrMask;
|
||||
|
||||
for (int i3 = 0; i3 < 512; i3++) {
|
||||
Descriptor leaf = TableAt(l3)[i3];
|
||||
if ((leaf & DescValid) == 0) continue;
|
||||
|
||||
// Write-combining/device pages are not the process's.
|
||||
if (((leaf & DescAttrMask) >> DescAttrShift) != AttrNormal) continue;
|
||||
|
||||
uint64_t pagePhys = leaf & DescAddrMask;
|
||||
if (pagePhys != 0) {
|
||||
Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
|
||||
}
|
||||
}
|
||||
|
||||
Memory::g_pfa->Free((void*)Memory::HHDM(l3));
|
||||
}
|
||||
|
||||
Memory::g_pfa->Free((void*)Memory::HHDM(l2));
|
||||
}
|
||||
|
||||
Memory::g_pfa->Free((void*)Memory::HHDM(l1));
|
||||
}
|
||||
}
|
||||
|
||||
void Paging::MapEfiRuntime(const montauk::boot::EfiInfo&) {
|
||||
NotImplemented("aarch64: Paging::MapEfiRuntime not implemented");
|
||||
bool Paging::IsUserRangeAccessible(std::uint64_t root, std::uint64_t virtualAddress,
|
||||
std::uint64_t size, bool requireWrite) {
|
||||
if (size == 0) return true;
|
||||
if (virtualAddress >= UserSpaceEnd) return false;
|
||||
if (virtualAddress + size < virtualAddress) return false;
|
||||
if (virtualAddress + size > UserSpaceEnd) return false;
|
||||
|
||||
uint64_t page = virtualAddress & ~0xFFFULL;
|
||||
uint64_t lastPage = (virtualAddress + size - 1) & ~0xFFFULL;
|
||||
for (;;) {
|
||||
volatile Descriptor* entry = WalkExisting(root, page);
|
||||
if (entry == nullptr) return false;
|
||||
Descriptor desc = *entry;
|
||||
if (!IsUserLeaf(desc)) return false;
|
||||
if (requireWrite && (desc & DescApReadOnly)) return false;
|
||||
if (page == lastPage) break;
|
||||
page += 0x1000;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -12,12 +12,32 @@
|
||||
|
||||
using namespace Kt;
|
||||
|
||||
extern "C" char ArchExceptionVectors[]; // Vectors.S
|
||||
|
||||
namespace Arch {
|
||||
|
||||
// SCTLR_EL1.WXN: writable memory is execute-never
|
||||
static constexpr uint64_t SctlrWxn = 1ULL << 19;
|
||||
|
||||
void EarlyInitialize() {
|
||||
// No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
|
||||
asm volatile("msr tpidr_el1, xzr");
|
||||
|
||||
// Run on SP_EL1 so exceptions taken from the kernel arrive on the
|
||||
// stack it is already using (Vectors.S has no SP_EL0 handlers).
|
||||
asm volatile("mov x9, sp\n\tmsr spsel, #1\n\tmov sp, x9" ::: "x9", "memory");
|
||||
|
||||
// The vectors need no allocation, so take over from the
|
||||
// bootloader's right away.
|
||||
asm volatile("msr vbar_el1, %0\n\tisb" :: "r"(ArchExceptionVectors) : "memory");
|
||||
|
||||
// Kernel mappings are RWX (Paging.cpp), so writable must not
|
||||
// imply execute-never.
|
||||
uint64_t sctlr;
|
||||
asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr));
|
||||
sctlr &= ~SctlrWxn;
|
||||
asm volatile("msr sctlr_el1, %0\n\tisb" :: "r"(sctlr));
|
||||
|
||||
// CPACR_EL1.FPEN = 0b11: FP/SIMD usable at EL0 and EL1 (the kernel
|
||||
// itself is built -mgeneral-regs-only; Context.S saves user state).
|
||||
uint64_t cpacr;
|
||||
@@ -26,10 +46,10 @@ namespace Arch {
|
||||
asm volatile("msr cpacr_el1, %0\n\tisb" :: "r"(cpacr));
|
||||
}
|
||||
|
||||
// TODO(aarch64): install the VBAR_EL1 vector table (EL1/EL0 synchronous
|
||||
// exceptions -> panic / SyscallDispatch, IRQ -> GIC -> ArchIrqDispatch).
|
||||
// Installed by EarlyInitialize.
|
||||
void InitializeTraps() {
|
||||
KernelLogStream(WARNING, "Arch") << "aarch64: exception vectors not implemented";
|
||||
KernelLogStream(OK, "Arch") << "Installed exception vectors at 0x" << base::hex
|
||||
<< (uint64_t)ArchExceptionVectors;
|
||||
}
|
||||
|
||||
// MAIR_EL1 attribute indices are chosen together with the page-table
|
||||
|
||||
@@ -9,13 +9,17 @@
|
||||
|
||||
namespace Arch {
|
||||
|
||||
// Saved by the EL0 synchronous-exception vector for `svc #0`:
|
||||
// x0-x5 carry the arguments and x8 the system-call number.
|
||||
// The exception frame Vectors.S saves for `svc #0` (the same layout as
|
||||
// Arch::PanicFrame): x0-x5 carry the arguments and x8 the system-call
|
||||
// number; the result is written back to x0.
|
||||
struct SyscallFrame {
|
||||
uint64_t arg1, arg2, arg3, arg4, arg5, arg6; // x0-x5
|
||||
uint64_t x6, x7;
|
||||
uint64_t syscall_nr; // x8
|
||||
uint64_t elr, spsr, sp_el0; // user PC, PSTATE, SP
|
||||
uint64_t x19_x30[12]; // callee-saved + LR
|
||||
uint64_t x9_x30[22];
|
||||
uint64_t sp_el0, elr, spsr, esr, far;
|
||||
};
|
||||
|
||||
static_assert(sizeof(SyscallFrame) == 288, "must match Vectors.S");
|
||||
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/*
|
||||
Vectors.S
|
||||
aarch64 exception vector table (VBAR_EL1)
|
||||
Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
/*
|
||||
* Every exception saves the same 288-byte frame on SP_EL1, laid out as
|
||||
* Arch::PanicFrame (InterruptFrame.hpp) and Arch::SyscallFrame:
|
||||
*
|
||||
* 0 x0 .. x30 (x30 at 240)
|
||||
* 248 interrupted SP (SP_EL0 from EL0, the pre-exception SP_EL1 from EL1)
|
||||
* 256 ELR_EL1
|
||||
* 264 SPSR_EL1
|
||||
* 272 ESR_EL1
|
||||
* 280 FAR_EL1
|
||||
*
|
||||
* The kernel runs on SP_EL1 (SPSel = 1, see Platform.cpp). From EL0,
|
||||
* SP_EL1 is the top of the thread's kernel stack: ArchEnterUserMode sets
|
||||
* it before the first ERET and every return to EL0 restores it.
|
||||
*/
|
||||
|
||||
#define FRAME_SIZE 288
|
||||
#define FRAME_SP 248
|
||||
#define FRAME_ELR 256
|
||||
#define FRAME_SPSR 264
|
||||
#define FRAME_ESR 272
|
||||
#define FRAME_FAR 280
|
||||
|
||||
.text
|
||||
|
||||
.macro SAVE_FRAME el
|
||||
sub sp, sp, #FRAME_SIZE
|
||||
stp x0, x1, [sp, #0]
|
||||
stp x2, x3, [sp, #16]
|
||||
stp x4, x5, [sp, #32]
|
||||
stp x6, x7, [sp, #48]
|
||||
stp x8, x9, [sp, #64]
|
||||
stp x10, x11, [sp, #80]
|
||||
stp x12, x13, [sp, #96]
|
||||
stp x14, x15, [sp, #112]
|
||||
stp x16, x17, [sp, #128]
|
||||
stp x18, x19, [sp, #144]
|
||||
stp x20, x21, [sp, #160]
|
||||
stp x22, x23, [sp, #176]
|
||||
stp x24, x25, [sp, #192]
|
||||
stp x26, x27, [sp, #208]
|
||||
stp x28, x29, [sp, #224]
|
||||
.if \el == 0
|
||||
mrs x21, sp_el0
|
||||
.else
|
||||
add x21, sp, #FRAME_SIZE
|
||||
.endif
|
||||
stp x30, x21, [sp, #240]
|
||||
mrs x22, elr_el1
|
||||
mrs x23, spsr_el1
|
||||
stp x22, x23, [sp, #FRAME_ELR]
|
||||
mrs x24, esr_el1
|
||||
mrs x25, far_el1
|
||||
stp x24, x25, [sp, #FRAME_ESR]
|
||||
.endm
|
||||
|
||||
/* Interrupts are masked again before ELR/SPSR are reloaded: an IRQ taken
|
||||
in between would overwrite them. */
|
||||
.macro RESTORE_FRAME el
|
||||
msr daifset, #2
|
||||
ldp x22, x23, [sp, #FRAME_ELR]
|
||||
msr elr_el1, x22
|
||||
msr spsr_el1, x23
|
||||
.if \el == 0
|
||||
ldr x21, [sp, #FRAME_SP]
|
||||
msr sp_el0, x21
|
||||
.endif
|
||||
ldp x0, x1, [sp, #0]
|
||||
ldp x2, x3, [sp, #16]
|
||||
ldp x4, x5, [sp, #32]
|
||||
ldp x6, x7, [sp, #48]
|
||||
ldp x8, x9, [sp, #64]
|
||||
ldp x10, x11, [sp, #80]
|
||||
ldp x12, x13, [sp, #96]
|
||||
ldp x14, x15, [sp, #112]
|
||||
ldp x16, x17, [sp, #128]
|
||||
ldp x18, x19, [sp, #144]
|
||||
ldp x20, x21, [sp, #160]
|
||||
ldp x22, x23, [sp, #176]
|
||||
ldp x24, x25, [sp, #192]
|
||||
ldp x26, x27, [sp, #208]
|
||||
ldp x28, x29, [sp, #224]
|
||||
ldr x30, [sp, #240]
|
||||
add sp, sp, #FRAME_SIZE
|
||||
eret
|
||||
.endm
|
||||
|
||||
/* Each vector slot is 128 bytes; the handlers live outside the table. */
|
||||
.macro VECTOR target
|
||||
.balign 0x80
|
||||
b \target
|
||||
.endm
|
||||
|
||||
.balign 0x800
|
||||
.global ArchExceptionVectors
|
||||
ArchExceptionVectors:
|
||||
/* Current EL with SP_EL0: the kernel never runs on SP_EL0. */
|
||||
VECTOR el1t_unexpected
|
||||
VECTOR el1t_unexpected
|
||||
VECTOR el1t_unexpected
|
||||
VECTOR el1t_unexpected
|
||||
/* Current EL with SP_ELx */
|
||||
VECTOR el1h_sync
|
||||
VECTOR el1h_irq
|
||||
VECTOR el1h_fiq
|
||||
VECTOR el1h_serror
|
||||
/* Lower EL, AArch64 */
|
||||
VECTOR el0_sync
|
||||
VECTOR el0_irq
|
||||
VECTOR el0_fiq
|
||||
VECTOR el0_serror
|
||||
/* Lower EL, AArch32: not supported */
|
||||
VECTOR el0_aarch32
|
||||
VECTOR el0_aarch32
|
||||
VECTOR el0_aarch32
|
||||
VECTOR el0_aarch32
|
||||
|
||||
/* ---- Exceptions from the kernel ---- */
|
||||
|
||||
el1t_unexpected:
|
||||
SAVE_FRAME 1
|
||||
mov x0, sp
|
||||
mov x1, #0
|
||||
bl ArchUnexpectedException
|
||||
|
||||
el1h_sync:
|
||||
SAVE_FRAME 1
|
||||
mov x0, sp
|
||||
mov x1, #0
|
||||
bl ArchSyncException
|
||||
RESTORE_FRAME 1
|
||||
|
||||
el1h_irq:
|
||||
SAVE_FRAME 1
|
||||
mov x0, sp
|
||||
mov x1, #0
|
||||
bl ArchIrqException
|
||||
RESTORE_FRAME 1
|
||||
|
||||
el1h_fiq:
|
||||
SAVE_FRAME 1
|
||||
mov x0, sp
|
||||
mov x1, #1
|
||||
bl ArchUnexpectedException
|
||||
|
||||
el1h_serror:
|
||||
SAVE_FRAME 1
|
||||
mov x0, sp
|
||||
mov x1, #2
|
||||
bl ArchUnexpectedException
|
||||
|
||||
/* ---- Exceptions from user mode ---- */
|
||||
|
||||
el0_sync:
|
||||
SAVE_FRAME 0
|
||||
lsr x9, x24, #26 /* ESR_EL1.EC */
|
||||
cmp x9, #0x15 /* SVC from AArch64 */
|
||||
b.ne 1f
|
||||
|
||||
msr daifclr, #2
|
||||
mov x0, sp
|
||||
bl SyscallDispatch
|
||||
str x0, [sp, #0]
|
||||
bl SchedExitIfKilled
|
||||
RESTORE_FRAME 0
|
||||
|
||||
1: mov x0, sp
|
||||
mov x1, #1
|
||||
bl ArchSyncException
|
||||
RESTORE_FRAME 0
|
||||
|
||||
el0_irq:
|
||||
SAVE_FRAME 0
|
||||
mov x0, sp
|
||||
mov x1, #1
|
||||
bl ArchIrqException
|
||||
RESTORE_FRAME 0
|
||||
|
||||
el0_fiq:
|
||||
SAVE_FRAME 0
|
||||
mov x0, sp
|
||||
mov x1, #1
|
||||
bl ArchUnexpectedException
|
||||
|
||||
el0_serror:
|
||||
SAVE_FRAME 0
|
||||
mov x0, sp
|
||||
mov x1, #2
|
||||
bl ArchUnexpectedException
|
||||
|
||||
el0_aarch32:
|
||||
SAVE_FRAME 0
|
||||
mov x0, sp
|
||||
mov x1, #3
|
||||
bl ArchUnexpectedException
|
||||
@@ -27,6 +27,7 @@
|
||||
#include <Pci/Pci.hpp>
|
||||
#include <Timekeeping/Timer.hpp>
|
||||
#include <Drivers/Init.hpp>
|
||||
#include <Drivers/Serial/Uart.hpp>
|
||||
#include <Graphics/Framebuffer.hpp>
|
||||
#include <Arch/Cpu.hpp>
|
||||
#include <Arch/Platform.hpp>
|
||||
@@ -111,6 +112,8 @@ extern "C" void kmain() {
|
||||
|
||||
Hal::ACPI g_acpi((Hal::ACPI::XSDP*)Memory::HHDM(boot.rsdpPhysical));
|
||||
|
||||
Drivers::Serial::InitializeConsole(g_acpi.GetXSDT());
|
||||
|
||||
if (g_acpi.GetXSDT() != nullptr) {
|
||||
Hal::AcpiShutdown::Initialize(g_acpi.GetXSDT());
|
||||
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Uart.cpp
|
||||
* Polled serial console from the ACPI SPCR table
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#include "Uart.hpp"
|
||||
#include <Terminal/Terminal.hpp>
|
||||
#include <CppLib/Stream.hpp>
|
||||
#include <Memory/HHDM.hpp>
|
||||
#include <Memory/Paging.hpp>
|
||||
#include <Io/IoPort.hpp>
|
||||
#include <Arch/Cpu.hpp>
|
||||
|
||||
using namespace Kt;
|
||||
|
||||
namespace Drivers::Serial {
|
||||
|
||||
struct GenericAddress {
|
||||
uint8_t AddressSpaceId; // 0 = system memory, 1 = system I/O
|
||||
uint8_t RegisterBitWidth;
|
||||
uint8_t RegisterBitOffset;
|
||||
uint8_t AccessSize; // 1 = byte, 2 = word, 3 = dword, 4 = qword
|
||||
uint64_t Address;
|
||||
} __attribute__((packed));
|
||||
|
||||
struct Spcr {
|
||||
Hal::ACPI::CommonSDTHeader Header;
|
||||
uint8_t InterfaceType;
|
||||
uint8_t Reserved[3];
|
||||
GenericAddress BaseAddress;
|
||||
uint8_t InterruptType;
|
||||
uint8_t Irq;
|
||||
uint32_t GlobalSystemInterrupt;
|
||||
uint8_t BaudRate;
|
||||
uint8_t Parity;
|
||||
uint8_t StopBits;
|
||||
uint8_t FlowControl;
|
||||
uint8_t TerminalType;
|
||||
} __attribute__((packed));
|
||||
|
||||
// SPCR / DBG2 serial interface subtypes
|
||||
static constexpr uint8_t Type16550 = 0x00;
|
||||
static constexpr uint8_t Type16450 = 0x01;
|
||||
static constexpr uint8_t TypePl011 = 0x03;
|
||||
static constexpr uint8_t TypeSbsa32 = 0x0D;
|
||||
static constexpr uint8_t TypeSbsaGeneric = 0x0E;
|
||||
static constexpr uint8_t TypeBcm2835 = 0x10;
|
||||
static constexpr uint8_t Type16550Generic = 0x12;
|
||||
|
||||
// PL011: data register, flag register (TX FIFO full)
|
||||
static constexpr uint64_t Pl011Dr = 0x00;
|
||||
static constexpr uint64_t Pl011Fr = 0x18;
|
||||
static constexpr uint32_t Pl011FrTxff = 1u << 5;
|
||||
|
||||
// 16550: transmit holding register, line status register (THR empty)
|
||||
static constexpr uint64_t Uart16550Thr = 0;
|
||||
static constexpr uint64_t Uart16550Lsr = 5;
|
||||
static constexpr uint8_t Uart16550LsrThre = 1u << 5;
|
||||
|
||||
// Bounded so a wedged or absent UART cannot hang the console.
|
||||
static constexpr int TxSpinLimit = 100000;
|
||||
|
||||
static volatile uint8_t* g_mmio = nullptr;
|
||||
static uint16_t g_port = 0;
|
||||
static uint32_t g_stride = 1;
|
||||
|
||||
static inline uint32_t Read32(uint64_t offset) {
|
||||
return *(volatile uint32_t*)(g_mmio + offset);
|
||||
}
|
||||
|
||||
static inline void Write32(uint64_t offset, uint32_t value) {
|
||||
*(volatile uint32_t*)(g_mmio + offset) = value;
|
||||
}
|
||||
|
||||
static void Pl011Putchar(char c) {
|
||||
for (int i = 0; i < TxSpinLimit && (Read32(Pl011Fr) & Pl011FrTxff); i++) {
|
||||
Arch::Pause();
|
||||
}
|
||||
Write32(Pl011Dr, (uint8_t)c);
|
||||
}
|
||||
|
||||
static uint8_t Mmio16550Read(uint64_t reg) {
|
||||
if (g_stride == 4) return (uint8_t)Read32(reg * 4);
|
||||
return g_mmio[reg];
|
||||
}
|
||||
|
||||
static void Mmio16550Write(uint64_t reg, uint8_t value) {
|
||||
if (g_stride == 4) Write32(reg * 4, value);
|
||||
else g_mmio[reg] = value;
|
||||
}
|
||||
|
||||
static void Mmio16550Putchar(char c) {
|
||||
for (int i = 0; i < TxSpinLimit && !(Mmio16550Read(Uart16550Lsr) & Uart16550LsrThre); i++) {
|
||||
Arch::Pause();
|
||||
}
|
||||
Mmio16550Write(Uart16550Thr, (uint8_t)c);
|
||||
}
|
||||
|
||||
static void Port16550Putchar(char c) {
|
||||
for (int i = 0; i < TxSpinLimit && !(Io::In8(g_port + Uart16550Lsr) & Uart16550LsrThre); i++) {
|
||||
Arch::Pause();
|
||||
}
|
||||
Io::Out8((uint8_t)c, g_port + Uart16550Thr);
|
||||
}
|
||||
|
||||
static Spcr* FindSpcr(Hal::ACPI::CommonSDTHeader* xsdt) {
|
||||
uint32_t entryCount = (xsdt->Length - sizeof(Hal::ACPI::CommonSDTHeader)) / 8;
|
||||
uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(Hal::ACPI::CommonSDTHeader));
|
||||
|
||||
for (uint32_t i = 0; i < entryCount; i++) {
|
||||
auto* header = (Hal::ACPI::CommonSDTHeader*)Memory::HHDM(entries[i]);
|
||||
const char* sig = header->Signature;
|
||||
if (sig[0] == 'S' && sig[1] == 'P' && sig[2] == 'C' && sig[3] == 'R') {
|
||||
return (Spcr*)header;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static volatile uint8_t* MapRegisters(uint64_t phys) {
|
||||
uint64_t page = phys & ~0xFFFULL;
|
||||
Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
|
||||
return (volatile uint8_t*)Memory::HHDM(phys);
|
||||
}
|
||||
|
||||
void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt) {
|
||||
if (xsdt == nullptr) return;
|
||||
|
||||
Spcr* spcr = FindSpcr(xsdt);
|
||||
if (spcr == nullptr || spcr->Header.Length < sizeof(Spcr)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const GenericAddress& reg = spcr->BaseAddress;
|
||||
if (reg.Address == 0) return;
|
||||
|
||||
SerialSink sink = nullptr;
|
||||
const char* name = "";
|
||||
bool isIo = reg.AddressSpaceId == 1;
|
||||
|
||||
switch (spcr->InterfaceType) {
|
||||
case TypePl011:
|
||||
case TypeSbsa32:
|
||||
case TypeSbsaGeneric:
|
||||
if (isIo) return;
|
||||
g_mmio = MapRegisters(reg.Address);
|
||||
sink = Pl011Putchar;
|
||||
name = "PL011";
|
||||
break;
|
||||
|
||||
case Type16550:
|
||||
case Type16450:
|
||||
case Type16550Generic:
|
||||
case TypeBcm2835:
|
||||
if (isIo) {
|
||||
if (!Io::HasPortIo) return;
|
||||
g_port = (uint16_t)reg.Address;
|
||||
sink = Port16550Putchar;
|
||||
} else {
|
||||
// The BCM2835 mini UART is a 16550 with 32-bit registers.
|
||||
g_stride = (reg.RegisterBitWidth == 32 || reg.AccessSize == 3
|
||||
|| spcr->InterfaceType == TypeBcm2835) ? 4 : 1;
|
||||
g_mmio = MapRegisters(reg.Address);
|
||||
sink = Mmio16550Putchar;
|
||||
}
|
||||
name = spcr->InterfaceType == TypeBcm2835 ? "BCM2835 mini UART" : "16550";
|
||||
break;
|
||||
|
||||
default:
|
||||
KernelLogStream(WARNING, "Serial") << "SPCR interface type 0x" << base::hex
|
||||
<< (uint64_t)spcr->InterfaceType << " not supported";
|
||||
return;
|
||||
}
|
||||
|
||||
AttachSerialConsole(sink);
|
||||
KernelLogStream(OK, "Serial") << "Console on " << name << " at " << (isIo ? "port " : "")
|
||||
<< "0x" << base::hex << reg.Address;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Uart.hpp
|
||||
* Polled serial console from the ACPI SPCR table
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <ACPI/ACPI.hpp>
|
||||
|
||||
namespace Drivers::Serial {
|
||||
|
||||
// Find the firmware's serial console (SPCR: PL011, SBSA generic UART,
|
||||
// 16550 or the BCM2835 mini UART) and mirror the kernel console to it.
|
||||
// Uses the baud rate the firmware configured. Needs the kernel page
|
||||
// tables (the registers are mapped uncached through the HHDM).
|
||||
void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt);
|
||||
|
||||
}
|
||||
@@ -187,7 +187,20 @@ namespace Pci {
|
||||
Io::Out32(value, ConfigDataPort);
|
||||
}
|
||||
|
||||
// Drivers reach config space through the Legacy* calls below. Without
|
||||
// an I/O address space (ARM) there is no CF8/CFC mechanism, so they use
|
||||
// ECAM there, at the access's own width (16/8-bit writes must not
|
||||
// read-modify-write a neighbouring RW1C register).
|
||||
static bool EcamCovers(uint8_t bus);
|
||||
|
||||
static bool UseEcamForLegacy(uint8_t bus) {
|
||||
return !Io::HasPortIo && EcamCovers(bus);
|
||||
}
|
||||
|
||||
uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
|
||||
if (UseEcamForLegacy(bus)) {
|
||||
return EcamRead32(g_ecamBase, bus, device, function, offset & 0xFC);
|
||||
}
|
||||
g_legacyConfigLock.Acquire();
|
||||
uint32_t value = LegacyRead32Unlocked(bus, device, function, offset);
|
||||
g_legacyConfigLock.Release();
|
||||
@@ -205,12 +218,20 @@ namespace Pci {
|
||||
}
|
||||
|
||||
void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) {
|
||||
if (UseEcamForLegacy(bus)) {
|
||||
EcamWrite32(g_ecamBase, bus, device, function, offset & 0xFC, value);
|
||||
return;
|
||||
}
|
||||
g_legacyConfigLock.Acquire();
|
||||
LegacyWrite32Unlocked(bus, device, function, offset, value);
|
||||
g_legacyConfigLock.Release();
|
||||
}
|
||||
|
||||
void LegacyWrite16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint16_t value) {
|
||||
if (UseEcamForLegacy(bus)) {
|
||||
EcamWrite16(g_ecamBase, bus, device, function, offset & 0xFE, value);
|
||||
return;
|
||||
}
|
||||
g_legacyConfigLock.Acquire();
|
||||
uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
|
||||
int shift = (offset & 2) * 8;
|
||||
@@ -221,6 +242,10 @@ namespace Pci {
|
||||
}
|
||||
|
||||
void LegacyWrite8(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint8_t value) {
|
||||
if (UseEcamForLegacy(bus)) {
|
||||
EcamWrite8(g_ecamBase, bus, device, function, offset, value);
|
||||
return;
|
||||
}
|
||||
g_legacyConfigLock.Acquire();
|
||||
uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
|
||||
int shift = (offset & 3) * 8;
|
||||
|
||||
@@ -37,6 +37,14 @@ namespace Kt {
|
||||
// ANSI escape sequence filter state
|
||||
static bool g_ansiEscape = false;
|
||||
|
||||
static SerialSink g_serialSink = nullptr;
|
||||
|
||||
static void SerialPutchar(char c) {
|
||||
if (g_serialSink == nullptr) return;
|
||||
if (c == '\n') g_serialSink('\r');
|
||||
g_serialSink(c);
|
||||
}
|
||||
|
||||
static void RingBufferAppend(char c) {
|
||||
// Strip ANSI escape sequences
|
||||
if (c == '\033') {
|
||||
@@ -209,17 +217,19 @@ namespace Kt {
|
||||
// framebuffer terminal below; storing it in the ring makes file
|
||||
// consumers treat one logical newline as two line breaks.
|
||||
RingBufferAppend(c);
|
||||
SerialPutchar(c);
|
||||
|
||||
if (g_suppressKernelLog) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Once a graphical app takes over, suppress ALL flanterm writes
|
||||
// (SYS_PRINT from user processes, etc.) to avoid painting text
|
||||
// over the GUI framebuffer.
|
||||
if (g_suppressKernelLog && g_kernelLogDepth == 0) {
|
||||
return;
|
||||
} else {
|
||||
// Once a graphical app takes over, suppress ALL flanterm writes
|
||||
// (SYS_PRINT from user processes, etc.) to avoid painting text
|
||||
// over the GUI framebuffer.
|
||||
if (g_suppressKernelLog) {
|
||||
return;
|
||||
}
|
||||
SerialPutchar(c);
|
||||
}
|
||||
|
||||
if (c == '\n') {
|
||||
@@ -235,6 +245,18 @@ namespace Kt {
|
||||
}
|
||||
}
|
||||
|
||||
void AttachSerialConsole(SerialSink sink) {
|
||||
g_termLock.Acquire();
|
||||
uint64_t start = g_klogCount < KLOG_BUF_SIZE ? 0 : g_klogHead;
|
||||
for (uint64_t i = 0; i < g_klogCount; i++) {
|
||||
char c = g_klogBuf[(start + i) % KLOG_BUF_SIZE];
|
||||
if (c == '\n') sink('\r');
|
||||
sink(c);
|
||||
}
|
||||
g_serialSink = sink;
|
||||
g_termLock.Release();
|
||||
}
|
||||
|
||||
void SuppressKernelLog() {
|
||||
g_suppressKernelLog = true;
|
||||
}
|
||||
|
||||
@@ -43,6 +43,12 @@ namespace Kt
|
||||
void Putchar(char c);
|
||||
void Print(const char *text);
|
||||
|
||||
// Mirror console output to a polled serial port (Drivers/Serial). The
|
||||
// sink receives the whole kernel log, also once the desktop suppresses
|
||||
// it on screen; what was logged before attaching is replayed first.
|
||||
using SerialSink = void (*)(char c);
|
||||
void AttachSerialConsole(SerialSink sink);
|
||||
|
||||
// Terminal mutex - protects flanterm from concurrent CPU access.
|
||||
// Uses Mutex (not Spinlock) to keep interrupts enabled while held,
|
||||
// preventing dropped PS/2 mouse/keyboard bytes during log output.
|
||||
|
||||
Reference in new issue
Block a user