daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: aarch64 port - fix build system obstacles, add exception handling, page tables

Browse Source
This commit is contained in:
daniel committed 2026-10-25 07:43:46 +01:00
1 parent aae4db01ad
commit 8b6379fb7e
19 files changed
+1204 -74
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

GNUmakefile
+37 -6
View File
@@ -10,6 +10,22 @@ QEMUFLAGS := -smp 4 -m 2G -d int -D qemu.log -no-reboot
override IMAGE_NAME := montauk-$(ARCH)
# Userspace exists only for x86_64 so far: other architectures get a
# ramdisk holding just the boot files.
ifeq ($(ARCH),x86_64)
KERNEL_ONLY ?= 0
else
KERNEL_ONLY ?= 1
endif
override EFI_LOADER_x86_64 := BOOTX64.EFI
override EFI_LOADER_aarch64 := BOOTAA64.EFI
override EFI_LOADER_riscv64 := BOOTRISCV64.EFI
override EFI_LOADER_loongarch64 := BOOTLOONGARCH64.EFI
# AArch64 UEFI firmware for QEMU (Debian/Ubuntu: qemu-efi-aarch64).
AARCH64_EFI_FIRMWARE ?= /usr/share/qemu-efi-aarch64/QEMU_EFI.fd
# Toolchain for building the 'limine' executable for the host.
HOST_CC := cc
HOST_CFLAGS := -g -O2 -pipe
@@ -51,29 +67,34 @@ run-hdd-x86_64: $(IMAGE_NAME).hdd
$(QEMUFLAGS)
.PHONY: run-aarch64
# The machine mirrors the Raspberry Pi 4/400: Cortex-A72 cores and a GICv2.
# The kernel console is mirrored to the PL011 (-serial stdio).
run-aarch64: $(IMAGE_NAME).iso
qemu-system-$(ARCH) \
-M virt \
-M virt,gic-version=2 \
-cpu cortex-a72 \
-device ramfb \
-device qemu-xhci,id=xhci \
-device usb-kbd,bus=xhci.0 \
-device usb-mouse,bus=xhci.0 \
-bios /usr/share/ovmf/OVMF.fd \
-bios $(AARCH64_EFI_FIRMWARE) \
-cdrom $(IMAGE_NAME).iso \
-serial stdio \
$(QEMUFLAGS)
.PHONY: run-hdd-aarch64
run-hdd-aarch64: $(IMAGE_NAME).hdd
qemu-system-$(ARCH) \
-M virt \
-M virt,gic-version=2 \
-cpu cortex-a72 \
-device ramfb \
-device qemu-xhci,id=xhci \
-device usb-kbd,bus=xhci.0 \
-device usb-mouse,bus=xhci.0 \
-bios /usr/share/ovmf/OVMF.fd \
-hda $(IMAGE_NAME).hdd \
-bios $(AARCH64_EFI_FIRMWARE) \
-drive if=none,id=hd0,format=raw,file=$(IMAGE_NAME).hdd \
-device virtio-blk-pci,drive=hd0 \
-serial stdio \
$(QEMUFLAGS)
.PHONY: run-riscv64
@@ -196,13 +217,23 @@ ports: programs
fi
.PHONY: ramdisk
ifeq ($(KERNEL_ONLY),1)
ramdisk: limine/limine kernel
rm -rf .kernel-only-root
mkdir -p .kernel-only-root/boot/limine
cp -v limine/$(EFI_LOADER_$(ARCH)) limine.conf .kernel-only-root/boot/limine/
cp -v kernel/bin-$(ARCH)/kernel .kernel-only-root/boot/
./scripts/mkramdisk.sh .kernel-only-root ramdisk.tar
rm -rf .kernel-only-root
else
ramdisk: limine/limine kernel programs
mkdir -p programs/bin/boot/limine
cp -v limine/BOOTX64.EFI programs/bin/boot/limine/
cp -v limine/$(EFI_LOADER_$(ARCH)) programs/bin/boot/limine/
cp -v limine.conf programs/bin/boot/limine/
cp -v kernel/bin-$(ARCH)/kernel programs/bin/boot/
rm -f programs/bin/boot/ramdisk.tar
./scripts/mkramdisk.sh programs/bin ramdisk.tar
endif
$(IMAGE_NAME).iso: limine/limine kernel ramdisk
rm -rf iso_root
docs/porting.md
+33 -10
View File
@@ -36,7 +36,7 @@ kernel/src/Arch/
Platform.hpp boot-sequence hooks, platform device list, reboot
CpuPower.hpp frequency scaling / thermal governor (may be no-ops)
x86_64/ the complete port
aarch64/ skeleton port
aarch64/ the port in progress (see below)
```
Other interfaces a port implements:
@@ -85,25 +85,48 @@ are `Cpu.hpp`, `IoPort.hpp`, `PerCpu.hpp`, `Context.hpp`, `Abi.hpp` and
- Interrupts are numbered IRQ slots (`Arch::IRQ_COUNT`); slot numbers for
MSI-capable drivers are fixed in the drivers.
## The aarch64 skeleton
## The aarch64 port
Real: CPU primitives (DAIF, WFI, `CNTVCT_EL0`, TLBI, TTBR0), per-CPU data via
`TPIDR_EL1`, the context switch and EL0 entry (`Context.S`), TLS through
`TPIDR_EL0`, the ELF/TLS/exit-stub ABI (syscall number in x8), the IRQ slot
dispatch table (`ArchIrqDispatch`), panic-frame printing.
dispatch table (`ArchIrqDispatch`), panic-frame printing, and:
- Exception vectors (`Vectors.S`, installed in `EarlyInitialize`). Every
exception saves one 288-byte frame that is both `Arch::PanicFrame` and
`Arch::SyscallFrame`. EL0 `svc` goes to `SyscallDispatch`; other
synchronous exceptions go to `ArchSyncException` (`Exceptions.cpp`), which
grows user stacks and lazy anonymous memory, kills faulting processes with
a crash report, and panics on kernel faults.
- Translation tables (`Paging.cpp`): 4 KiB granule, 48-bit VAs. The kernel
half lives in its own TTBR1 table; `KernelRoot()` is the TTBR0 table of
kernel contexts (EFI identity mappings only, so null pointers fault). The
HHDM maps only the memory map's RAM and firmware regions, never the holes
between them (device space on ARM). MAIR: 0 Normal WB, 1 Normal NC
(`MapWC`), 2 Device-nGnRE (`MapMMIO`).
Not yet written (each is marked `TODO(aarch64)`):
- `Paging.cpp`: translation tables and MAIR (every method panics).
- Exception vectors (`VBAR_EL1`): synchronous faults, `svc`, IRQs.
- GIC driver: SPI routing, SGIs for IPIs, MSIs via ITS/GICv2m
(`ComposeMsi` returns false, so drivers fall back to INTx/polling).
- Generic timer tick (`Arch::Timer::InitializeBsp` returns false).
(`ComposeMsi` returns false, so drivers fall back to INTx/polling). IRQs
reach `ArchIrqException`, which only logs them.
- Generic timer tick (`Arch::Timer::InitializeBsp` returns false). Boot
currently stops in the first driver that waits on `GetMilliseconds()`
(xHCI port power-up).
- Secondary CPU start-up, PSCI reset and suspend.
Building it needs `clang` and `ld.lld` (`make kernel ARCH=aarch64`;
`KERNEL_LD=` points at a different `ld.lld`, e.g. rustup's
`lib/rustlib/x86_64-unknown-linux-gnu/bin/gcc-ld/ld.lld`).
Building it needs `clang` and `ld.lld`; without a system `ld.lld` the
kernel makefile uses the one rustup ships
(`~/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld`), and `KERNEL_LD=`
overrides both. `make run ARCH=aarch64` boots a kernel-only image (userspace
is x86_64-only so far) on QEMU `virt` with a GICv2 and Cortex-A72 cores, the
Raspberry Pi 4/400's configuration; it needs `qemu-system-aarch64` and
`qemu-efi-aarch64`.
The kernel console is mirrored to the serial port the ACPI SPCR table names
(PL011, SBSA UART, 16550 or the BCM2835 mini UART; `Drivers/Serial/Uart.cpp`),
including the log written before the port was found. On QEMU that is
`-serial stdio`.
## Adding another architecture
kernel/GNUmakefile
+3 -1
View File
@@ -291,7 +291,9 @@ cc-runtime-$(ARCH)/cc-runtime.a: GNUmakefile cc-runtime/*
# so a clang-built non-x86 kernel is linked by ld.lld directly.
ifneq ($(ARCH),x86_64)
ifeq ($(CXX_IS_CLANG),1)
KERNEL_LD ?= ld.lld
# Without a system lld, fall back to the one rustup ships.
KERNEL_LD ?= $(firstword $(shell command -v ld.lld) \
$(wildcard $(HOME)/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld))
override LD_EMULATION_aarch64 := aarch64elf
override LD_EMULATION_riscv64 := elf64lriscv
override LD_EMULATION_loongarch64 := elf64loongarch
kernel/src/Api/BuildNo.hpp
+1 -1
View File
@@ -12,4 +12,4 @@
#pragma once
#define MONTAUK_BUILD_NUMBER 26
#define MONTAUK_BUILD_NUMBER 29
kernel/src/Arch/aarch64/Cpu.hpp
+2 -1
View File
@@ -92,9 +92,10 @@ namespace Arch {
asm volatile("dsb sy\n\tisb" ::: "memory");
}
// The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0.
inline void FlushTlbPage(uint64_t virtualAddress) {
asm volatile("dsb ishst\n\ttlbi vaae1is, %0\n\tdsb ish\n\tisb"
:: "r"(virtualAddress >> 12) : "memory");
:: "r"((virtualAddress >> 12) & ((1ULL << 44) - 1)) : "memory");
}
inline void FlushTlbAll() {
kernel/src/Arch/aarch64/Exceptions.cpp
+184
View File
@@ -0,0 +1,184 @@
/*
* Exceptions.cpp
* aarch64 synchronous exceptions: faults, user crashes, kernel panics
* Copyright (c) 2026 Daniel Hammer
*/
#include <Arch/aarch64/InterruptFrame.hpp>
#include <Arch/Cpu.hpp>
#include <Arch/Smp.hpp>
#include <Common/Panic.hpp>
#include <Common/TerminationCodes.hpp>
#include <CppLib/Stream.hpp>
#include <Memory/Paging.hpp>
#include <Sched/Scheduler.hpp>
#include <Sched/CrashReport.hpp>
#include <Api/Heap.hpp>
#include <Timekeeping/Timer.hpp>
using namespace Kt;
namespace {
// ESR_EL1.EC exception classes
constexpr uint64_t EcUnknown = 0x00;
constexpr uint64_t EcIllegalState = 0x0E;
constexpr uint64_t EcInstAbortLower = 0x20;
constexpr uint64_t EcInstAbortSame = 0x21;
constexpr uint64_t EcPcAlignment = 0x22;
constexpr uint64_t EcDataAbortLower = 0x24;
constexpr uint64_t EcDataAbortSame = 0x25;
constexpr uint64_t EcSpAlignment = 0x26;
constexpr uint64_t EcFpException = 0x2C;
constexpr uint64_t EcBreakpointLower = 0x30;
constexpr uint64_t EcBrk = 0x3C;
constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL;
uint64_t ExceptionClass(const Arch::PanicFrame* frame) {
return (frame->esr >> 26) & 0x3F;
}
bool IsAbort(uint64_t ec) {
return ec == EcInstAbortLower || ec == EcInstAbortSame
|| ec == EcDataAbortLower || ec == EcDataAbortSame;
}
// Fault status code (ISS[5:0]) of an abort: translation faults mean
// the page is not mapped, permission faults that it is.
bool IsTranslationFault(uint64_t esr) {
uint64_t fsc = esr & 0x3F;
return fsc >= 0x04 && fsc <= 0x07;
}
bool IsPermissionFault(uint64_t esr) {
uint64_t fsc = esr & 0x3F;
return fsc >= 0x0C && fsc <= 0x0F;
}
const char* ExceptionName(uint64_t ec) {
switch (ec) {
case EcUnknown: return "Undefined Instruction";
case EcIllegalState: return "Illegal Execution State";
case EcInstAbortLower:
case EcInstAbortSame: return "Instruction Abort";
case EcPcAlignment: return "PC Alignment Fault";
case EcDataAbortLower:
case EcDataAbortSame: return "Data Abort";
case EcSpAlignment: return "SP Alignment Fault";
case EcFpException: return "Floating-Point Exception";
case EcBrk: return "Breakpoint (BRK)";
default:
if (ec >= EcBreakpointLower && ec <= 0x35) return "Debug Exception";
return "Synchronous Exception";
}
}
TerminationCodes TerminationFromClass(uint64_t ec) {
if (ec == EcUnknown || ec == EcIllegalState) return TerminatedIllegalInstruction;
if (ec == EcBrk || (ec >= EcBreakpointLower && ec <= 0x35)) return TerminatedBreakpoint;
if (ec == EcFpException) return TerminatedArithmeticError;
return TerminatedAccessViolation;
}
template<size_t N>
void CopyName(char (&dst)[N], const char* src) {
size_t i = 0;
for (; i + 1 < N && src[i]; i++) dst[i] = src[i];
dst[i] = '\0';
}
// Kill the faulting process with a crash report. Never returns.
[[noreturn]] void TerminateUserProcess(Arch::PanicFrame* frame, uint64_t ec) {
// Teardown can wait on devices and other CPUs; keep the clock running.
Arch::EnableInterrupts();
auto* proc = Sched::GetCurrentProcessPtr();
const char* name = ExceptionName(ec);
KernelLogStream(ERROR, "Exception") << name << " in process \"" << proc->name
<< "\" (pid " << proc->pid << ", ESR 0x" << base::hex << frame->esr
<< ", PC 0x" << frame->elr << ") - process terminated";
CrashReport::Report rep{};
rep.valid = true;
rep.pid = proc->pid;
CopyName(rep.processName, proc->name);
rep.exceptionVector = (uint8_t)ec;
CopyName(rep.exceptionName, name);
rep.instructionPointer = frame->elr;
rep.flags = frame->spsr;
rep.stackPointer = frame->sp;
constexpr uint64_t snapshotBytes = CrashReport::StackSnapshotWords * sizeof(uint64_t);
if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pageTableRoot, frame->sp,
snapshotBytes, false)) {
auto* userStack = (const volatile uint64_t*)frame->sp;
for (int word = 0; word < CrashReport::StackSnapshotWords; word++)
rep.stackWords[word] = userStack[word];
rep.stackWordCount = CrashReport::StackSnapshotWords;
}
if (IsAbort(ec)) {
rep.faultingAddress = frame->far;
rep.pfPresent = IsPermissionFault(frame->esr);
rep.pfWrite = ec == EcDataAbortLower && (frame->esr & (1ULL << 6)) != 0;
rep.pfUser = 1;
rep.pfInstructionFetch = ec == EcInstAbortLower;
}
rep.timestampTick = Timekeeping::GetTicks();
CrashReport::AddReport(rep);
Sched::SpawnCrashPad(proc->pid);
// As on x86: a sibling thread's fault must end the whole process.
int code = TerminationFromClass(ec);
if (Sched::GetCurrentThreadPtr() != proc) {
Sched::KillProcess(proc->pid, code);
} else {
Sched::SetProcessExitCode(proc->pid, code);
}
Sched::ExitProcess();
__builtin_unreachable();
}
// A not-present user page: anonymous memory mapped lazily or the user
// stack growing. Also reached from the kernel touching user buffers.
bool TryResolveUserFault(const Arch::PanicFrame* frame, uint64_t ec) {
if (!IsAbort(ec) || !IsTranslationFault(frame->esr)) return false;
if (frame->far >= UserSpaceEnd) return false;
auto* cpu = Smp::TryGetCurrentCpuData();
if (cpu == nullptr || cpu->currentSlot < 0 || Sched::GetCurrentPid() < 0) return false;
bool isData = ec == EcDataAbortLower || ec == EcDataAbortSame;
Memory::VMM::PageFault fault{false, isData && (frame->esr & (1ULL << 6)) != 0, !isData};
return montauk::abi::TryHandleAnonymousPageFault(frame->far, fault)
|| Sched::TryGrowUserStack(frame->far);
}
}
extern "C" void ArchSyncException(Arch::PanicFrame* frame, bool fromUser) {
uint64_t ec = ExceptionClass(frame);
if (TryResolveUserFault(frame, ec)) {
return;
}
if (fromUser && Sched::GetCurrentPid() >= 0) {
TerminateUserProcess(frame, ec);
}
Panic(ExceptionName(ec), frame);
}
// FIQs are never routed, SErrors are asynchronous external aborts, and the
// kernel neither runs on SP_EL0 nor supports AArch32 user code.
extern "C" void ArchUnexpectedException(Arch::PanicFrame* frame, uint64_t kind) {
static const char* const names[] = {
"Exception on SP_EL0", "Unexpected FIQ", "SError (asynchronous abort)",
"AArch32 exception",
};
Panic(kind < 4 ? names[kind] : "Unexpected exception", frame);
}
kernel/src/Arch/aarch64/InterruptFrame.hpp
+3 -1
View File
@@ -9,7 +9,7 @@
namespace Arch {
// What the exception vectors save before calling into C++.
// What the exception vectors (Vectors.S) save before calling into C++.
struct PanicFrame {
uint64_t x[31]; // x0-x30
uint64_t sp; // interrupted SP (SP_EL0 for EL0, else SP_EL1)
@@ -19,4 +19,6 @@ namespace Arch {
uint64_t far; // FAR_EL1: faulting address
};
static_assert(sizeof(PanicFrame) == 288, "must match Vectors.S");
}
kernel/src/Arch/aarch64/Interrupts.cpp
+11
View File
@@ -45,6 +45,17 @@ namespace Arch {
}
// IRQ exception vector (Vectors.S).
// TODO(aarch64): acknowledge through the GIC CPU interface, translate the
// INTID to a slot and call ArchIrqDispatch.
extern "C" void ArchIrqException(void*, bool) {
static bool reported = false;
if (!reported) {
reported = true;
Kt::KernelLogStream(Kt::WARNING, "IRQ") << "aarch64: IRQ taken with no interrupt controller driver";
}
}
extern "C" void ArchIrqDispatch(uint64_t irq, bool fromUser) {
if (irq < Arch::IRQ_COUNT && Arch::g_irqHandlers[irq] != nullptr) {
Arch::g_irqHandlers[irq]((uint8_t)irq, fromUser);
kernel/src/Arch/aarch64/Paging.cpp
+436 -42
View File
@@ -5,29 +5,67 @@
*/
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Memory/HHDM.hpp>
#include <Arch/Cpu.hpp>
#include <Common/Panic.hpp>
#include <CppLib/Spinlock.hpp>
// TODO(aarch64): 4 KiB-granule, 48-bit VA translation tables. The kernel
// half lives in TTBR1_EL1 and is never switched; "address space roots"
// handed to the generic kernel are TTBR0_EL1 tables, so
// CreateUserAddressSpace has no kernel half to copy. MAIR_EL1 needs
// Normal-WB, Normal-NC (MapWC) and Device-nGnRE (MapMMIO) indices.
// 4 KiB granule, 48-bit virtual addresses, four levels (0-3). The kernel
// half is translated through TTBR1_EL1 (g_kernelHalfRoot), which never
// changes after Init. The "address space roots" the generic kernel switches
// between are TTBR0_EL1 tables: a process's root holds only its own lower
// half, and KernelRoot() is the TTBR0 table of kernel contexts, empty apart
// from the EFI runtime identity mappings (so null pointers fault).
//
// The HHDM covers only what the memory map lists as RAM or firmware data:
// unlike on x86, the holes between are device space on ARM machines, and a
// cacheable mapping of device registers invites speculative accesses.
namespace Memory::VMM {
Paging* g_paging = nullptr;
static std::uint64_t g_hhdmLimit = 0;
static std::uint64_t g_kernelHalfRoot = 0;
std::uint64_t Paging::HhdmLimit() { return g_hhdmLimit; }
// Translation table descriptor bits (VMSAv8-64, stage 1)
static constexpr uint64_t DescValid = 1ULL << 0;
static constexpr uint64_t DescTable = 1ULL << 1; // table (levels 0-2) or page (level 3)
static constexpr uint64_t DescAttrShift = 2; // AttrIndx[2:0], into MAIR_EL1
static constexpr uint64_t DescAttrMask = 7ULL << DescAttrShift;
static constexpr uint64_t DescApUser = 1ULL << 6; // AP[1]: EL0 may access
static constexpr uint64_t DescApReadOnly = 1ULL << 7; // AP[2]
static constexpr uint64_t DescShInner = 3ULL << 8;
static constexpr uint64_t DescAf = 1ULL << 10; // access flag (no AF faults)
static constexpr uint64_t DescNotGlobal = 1ULL << 11;
static constexpr uint64_t DescPxn = 1ULL << 53;
static constexpr uint64_t DescUxn = 1ULL << 54;
static constexpr uint64_t DescAddrMask = 0x0000FFFFFFFFF000ULL;
// MAIR_EL1 attribute indices
static constexpr uint64_t AttrNormal = 0; // write-back, read/write-allocate
static constexpr uint64_t AttrNormalNc = 1; // non-cacheable (write-combining)
static constexpr uint64_t AttrDevice = 2; // Device-nGnRE
static constexpr uint64_t MairValue = 0xFFULL << (8 * AttrNormal)
| 0x44ULL << (8 * AttrNormalNc)
| 0x04ULL << (8 * AttrDevice);
static constexpr uint64_t KernelHalfBase = 0xFFFF000000000000ULL;
static constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL;
// EFI memory types whose runtime regions are device registers
static constexpr uint32_t EfiMemoryMappedIo = 11;
static constexpr uint32_t EfiMemoryMappedIoPortSpace = 12;
static kcp::Mutex pagingLock;
void LockUserPaging() { pagingLock.Acquire(); }
void UnlockUserPaging() { pagingLock.Release(); }
[[noreturn]] static void NotImplemented(const char* what) {
Panic(what, nullptr);
}
void ActivateKernelAddressSpace() {
Arch::WritePageTableRoot(g_paging->KernelRoot());
}
@@ -36,75 +74,431 @@ namespace Memory::VMM {
Arch::FlushTlbAll();
}
using Descriptor = uint64_t;
static inline volatile Descriptor* TableAt(uint64_t phys) {
return (volatile Descriptor*)Memory::HHDM(phys);
}
static inline uint64_t Index(uint64_t va, int level) {
return (va >> (39 - 9 * level)) & 0x1FF;
}
// Order descriptor writes before the table walker (and later
// instructions) can observe them.
static inline void PublishTables() {
asm volatile("dsb ishst\n\tisb" ::: "memory");
}
static inline uint64_t ReadTtbr1() {
uint64_t ttbr;
asm volatile("mrs %0, ttbr1_el1" : "=r"(ttbr));
return ttbr & Arch::TtbrAddressMask;
}
// The level-3 descriptor for `va`, creating intermediate tables.
// Returns nullptr when out of memory or when a block mapping covers
// `va` (only the bootloader's tables contain blocks).
static volatile Descriptor* WalkCreate(uint64_t root, uint64_t va) {
uint64_t table = root;
for (int level = 0; level < 3; level++) {
volatile Descriptor* entry = &TableAt(table)[Index(va, level)];
Descriptor desc = *entry;
if ((desc & DescValid) == 0) {
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) return nullptr;
table = Memory::SubHHDM(page);
// The zeroed table must be visible before the walker can
// reach it.
asm volatile("dsb ishst" ::: "memory");
*entry = table | DescTable | DescValid;
} else if ((desc & DescTable) == 0) {
return nullptr;
} else {
table = desc & DescAddrMask;
}
}
return &TableAt(table)[Index(va, 3)];
}
// The level-3 descriptor for `va`, or nullptr if a table is missing.
static volatile Descriptor* WalkExisting(uint64_t root, uint64_t va) {
uint64_t table = root;
for (int level = 0; level < 3; level++) {
Descriptor desc = TableAt(table)[Index(va, level)];
if ((desc & (DescValid | DescTable)) != (DescValid | DescTable)) return nullptr;
table = desc & DescAddrMask;
}
return &TableAt(table)[Index(va, 3)];
}
// Install a leaf. Replacing a live translation (other address or
// attributes) needs break-before-make: invalidate, flush, then write.
static void SetLeaf(volatile Descriptor* entry, Descriptor desc, uint64_t va) {
Descriptor old = *entry;
if (old == desc) return;
if (old & DescValid) {
*entry = 0;
Arch::FlushTlbPage(va);
}
*entry = desc;
PublishTables();
}
static Descriptor KernelLeaf(uint64_t phys, uint64_t attr) {
Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
| (attr << DescAttrShift) | DescUxn;
// Kernel memory stays executable, as on x86; device memory never is.
if (attr != AttrNormal) desc |= DescPxn;
return desc;
}
static Descriptor UserLeaf(uint64_t phys, uint64_t attr, bool writable, bool executable) {
Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
| DescNotGlobal | (attr << DescAttrShift) | DescApUser | DescPxn;
if (!writable) desc |= DescApReadOnly;
if (!executable) desc |= DescUxn;
return desc;
}
static inline bool IsUserLeaf(Descriptor desc) {
return (desc & (DescValid | DescTable)) == (DescValid | DescTable) && (desc & DescApUser);
}
// Translate through a table tree that may contain block mappings (the
// bootloader's). Returns 0 if unmapped.
static uint64_t Translate(uint64_t root, uint64_t va) {
uint64_t table = root;
for (int level = 0; level <= 3; level++) {
Descriptor desc = TableAt(table)[Index(va, level)];
if ((desc & DescValid) == 0) return 0;
uint64_t span = 1ULL << (39 - 9 * level);
bool leaf = level == 3 || (desc & DescTable) == 0;
if (leaf) {
if (level == 0) return 0; // no level-0 blocks with 4 KiB granules
return ((desc & DescAddrMask) & ~(span - 1)) | (va & (span - 1));
}
table = desc & DescAddrMask;
}
return 0;
}
void Paging::MapEfiRuntime(const montauk::boot::EfiInfo& efi) {
if (efi.memoryMap == nullptr || efi.descriptorSize == 0) return;
struct EfiMemDesc {
uint32_t Type;
uint64_t PhysicalStart;
uint64_t VirtualStart;
uint64_t NumberOfPages;
uint64_t Attribute;
};
static constexpr uint64_t EFI_MEMORY_RUNTIME = 0x8000000000000000ULL;
auto* base = (uint8_t*)efi.memoryMap;
uint64_t count = efi.memoryMapSize / efi.descriptorSize;
for (uint64_t i = 0; i < count; i++) {
auto* desc = (EfiMemDesc*)(base + i * efi.descriptorSize);
if ((desc->Attribute & EFI_MEMORY_RUNTIME) == 0) continue;
bool device = desc->Type == EfiMemoryMappedIo || desc->Type == EfiMemoryMappedIoPortSpace;
for (uint64_t p = 0; p < desc->NumberOfPages; p++) {
uint64_t addr = desc->PhysicalStart + p * 0x1000;
// Firmware code runs at its HHDM alias (Efi::Init calls the
// services there) and reaches its data at physical addresses.
// Runtime regions are reserved memory, outside the HHDM.
if (device) {
MapMMIO(addr, addr);
MapMMIO(addr, Memory::HHDM(addr));
} else {
Map(addr, addr);
Map(addr, Memory::HHDM(addr));
}
}
}
FlushTLB();
}
Paging::Paging() {
NotImplemented("aarch64: virtual memory not implemented");
rootPhys = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
g_kernelHalfRoot = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
}
void Paging::Init(std::uint64_t, std::uint64_t, const montauk::boot::MemoryMap&,
const montauk::boot::Framebuffer&) {
NotImplemented("aarch64: Paging::Init not implemented");
// Kernel mappings of lower-half addresses (EFI identity mappings) go to
// the kernel contexts' TTBR0 table.
static uint64_t KernelRootFor(uint64_t kernelContextRoot, uint64_t va) {
return va >= KernelHalfBase ? g_kernelHalfRoot : kernelContextRoot;
}
std::uint64_t Paging::HhdmLimit() { return 0; }
void Paging::Map(std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::Map not implemented");
void Paging::Init(std::uint64_t kernelBaseVirt, std::uint64_t kernelSize, const montauk::boot::MemoryMap& memMap,
const montauk::boot::Framebuffer& framebuffer) {
if (Memory::HHDMBase < KernelHalfBase) {
Panic("aarch64: HHDM outside the 48-bit kernel half", nullptr);
}
void Paging::MapMMIO(std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::MapMMIO not implemented");
uint64_t bootTtbr1 = ReadTtbr1();
uint64_t bootTcr, bootMair;
asm volatile("mrs %0, tcr_el1" : "=r"(bootTcr));
asm volatile("mrs %0, mair_el1" : "=r"(bootMair));
Kt::KernelLogStream(Kt::DEBUG, "VMM") << "Bootloader TCR 0x" << base::hex << bootTcr
<< ", MAIR 0x" << bootMair << ", TTBR1 0x" << bootTtbr1;
// The kernel image, at the physical pages the bootloader loaded it to.
for (uint64_t va = kernelBaseVirt; va < kernelBaseVirt + kernelSize; va += 0x1000) {
uint64_t phys = Translate(bootTtbr1, va);
if (phys == 0) Panic("aarch64: kernel page missing from the bootloader's tables", nullptr);
Map(phys, va);
}
void Paging::MapWC(std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::MapWC not implemented");
// HHDM: every region that is real backing store (see the top of
// this file). The limit is published for the AML interpreter.
uint64_t maxPhysAddr = 0;
for (size_t i = 0; i < memMap.count; i++) {
const auto& region = memMap.regions[i];
if (region.kind == montauk::boot::MemoryKind::Reserved
|| region.kind == montauk::boot::MemoryKind::Unknown) {
continue;
}
void Paging::MapUser(std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::MapUser not implemented");
uint64_t start = region.base & ~0xFFFULL;
uint64_t end = (region.base + region.length + 0xFFF) & ~0xFFFULL;
bool framebufferRegion = region.kind == montauk::boot::MemoryKind::Framebuffer;
for (uint64_t phys = start; phys < end; phys += 0x1000) {
if (framebufferRegion) MapWC(phys, Memory::HHDM(phys));
else Map(phys, Memory::HHDM(phys));
}
if (end > maxPhysAddr) maxPhysAddr = end;
}
g_hhdmLimit = maxPhysAddr;
// The framebuffer may not be listed in the memory map at all.
if (framebuffer.address != 0) {
uint64_t fbPhys = Memory::SubHHDM(framebuffer.address) & ~0xFFFULL;
uint64_t fbPages = (framebuffer.height * framebuffer.pitch + 0xFFF) / 0x1000;
for (uint64_t p = 0; p < fbPages; p++) {
uint64_t phys = fbPhys + p * 0x1000;
MapWC(phys, Memory::HHDM(phys));
}
}
std::uint64_t Paging::GetPhysAddr(std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::GetPhysAddr not implemented");
// 48-bit VAs in both halves, 4 KiB granules, write-back walks of
// inner-shareable tables; the physical size is what the CPU has.
uint64_t mmfr0;
asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0));
uint64_t ips = mmfr0 & 0xF;
if (ips > 5) ips = 5; // 52-bit output needs LPA2 descriptors
uint64_t tcr = 16ULL // T0SZ
| 1ULL << 8 | 1ULL << 10 // IRGN0, ORGN0: write-back
| 3ULL << 12 // SH0: inner shareable
| 0ULL << 14 // TG0: 4 KiB
| 16ULL << 16 // T1SZ
| 1ULL << 24 | 1ULL << 26 // IRGN1, ORGN1
| 3ULL << 28 // SH1
| 2ULL << 30 // TG1: 4 KiB
| ips << 32; // IPS
// The new tables map the code running here at the same address, so
// the switch is a register write followed by a full invalidate.
asm volatile(
"dsb ish\n\t"
"msr mair_el1, %0\n\t"
"msr tcr_el1, %1\n\t"
"isb\n\t"
"msr ttbr1_el1, %2\n\t"
"msr ttbr0_el1, %3\n\t"
"isb\n\t"
"tlbi vmalle1is\n\t"
"dsb ish\n\t"
"isb"
:: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(rootPhys) : "memory");
Kt::KernelLogStream(Kt::OK, "VMM") << "Switched to kernel translation tables (HHDM limit 0x"
<< base::hex << g_hhdmLimit << ")";
}
static void MapKernelPage(uint64_t root, uint64_t phys, uint64_t va, uint64_t attr, const char* what) {
if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
Panic(what, nullptr);
}
volatile Descriptor* entry = WalkCreate(KernelRootFor(root, va), va);
if (entry == nullptr) {
Panic("OOM in Paging (kernel page table allocation)", nullptr);
}
SetLeaf(entry, KernelLeaf(phys, attr), va);
}
void Paging::Map(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormal,
"Value that isn't page-aligned passed as address to Paging::Map!");
}
void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrDevice,
"Value that isn't page-aligned passed as address to Paging::MapMMIO!");
}
void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormalNc,
"Value that isn't page-aligned passed as address to Paging::MapWC!");
}
void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
Panic("Value that isn't page-aligned passed as address to Paging::MapUser!", nullptr);
}
volatile Descriptor* entry = WalkCreate(rootPhys, virtualAddress);
if (entry == nullptr) {
Panic("OOM in Paging::MapUser (page table allocation)", nullptr);
}
SetLeaf(entry, UserLeaf(physicalAddress, AttrNormal, true, true), virtualAddress);
}
std::uint64_t Paging::GetPhysAddr(std::uint64_t root, std::uint64_t virtualAddress) {
if (virtualAddress >= KernelHalfBase) {
root = g_kernelHalfRoot;
}
return Translate(root, virtualAddress);
}
std::uint64_t Paging::GetPhysAddr(std::uint64_t virtualAddress) {
return GetPhysAddr(rootPhys, virtualAddress);
}
// The kernel half lives in TTBR1, so a new address space starts empty.
std::uint64_t Paging::CreateUserAddressSpace() {
NotImplemented("aarch64: Paging::CreateUserAddressSpace not implemented");
void* page = Memory::g_pfa->AllocateZeroed();
if (page == nullptr) return 0;
return Memory::SubHHDM(page);
}
bool Paging::MapUserIn(std::uint64_t, std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::MapUserIn not implemented");
bool Paging::MapUserIn(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
return MapUserInPermissions(root, physicalAddress, virtualAddress, true, true);
}
bool Paging::MapUserInPermissions(std::uint64_t, std::uint64_t, std::uint64_t, bool, bool) {
NotImplemented("aarch64: Paging::MapUserInPermissions not implemented");
static bool MapUserLeaf(uint64_t root, uint64_t phys, uint64_t va, Descriptor desc, const char* what) {
pagingLock.Acquire();
if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
pagingLock.Release();
Panic(what, nullptr);
}
volatile Descriptor* entry = WalkCreate(root, va);
if (entry == nullptr) {
pagingLock.Release();
return false;
}
SetLeaf(entry, desc, va);
pagingLock.Release();
return true;
}
bool Paging::ProtectUserIn(std::uint64_t, std::uint64_t, bool, bool) {
NotImplemented("aarch64: Paging::ProtectUserIn not implemented");
bool Paging::MapUserInPermissions(std::uint64_t root, std::uint64_t physicalAddress,
std::uint64_t virtualAddress, bool writable, bool executable) {
return MapUserLeaf(root, physicalAddress, virtualAddress,
UserLeaf(physicalAddress, AttrNormal, writable, executable),
"Non-aligned address in Paging::MapUserIn!");
}
bool Paging::MapUserInWC(std::uint64_t, std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::MapUserInWC not implemented");
bool Paging::MapUserInWC(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
return MapUserLeaf(root, physicalAddress, virtualAddress,
UserLeaf(physicalAddress, AttrNormalNc, true, false),
"Non-aligned address in Paging::MapUserInWC!");
}
void Paging::UnmapUserIn(std::uint64_t, std::uint64_t) {
NotImplemented("aarch64: Paging::UnmapUserIn not implemented");
// Permission changes need no break-before-make, only the TLB flush.
bool Paging::ProtectUserIn(std::uint64_t root, std::uint64_t virtualAddress,
bool writable, bool executable) {
if ((virtualAddress & 0xFFFULL) != 0) return false;
pagingLock.Acquire();
volatile Descriptor* entry = WalkExisting(root, virtualAddress);
if (entry == nullptr || !IsUserLeaf(*entry)) {
pagingLock.Release();
return false;
}
void Paging::FreeUserHalf(std::uint64_t) {
NotImplemented("aarch64: Paging::FreeUserHalf not implemented");
Descriptor desc = *entry;
if (writable) desc &= ~DescApReadOnly;
else desc |= DescApReadOnly;
if (executable) desc &= ~DescUxn;
else desc |= DescUxn;
*entry = desc;
Arch::FlushTlbPage(virtualAddress);
pagingLock.Release();
return true;
}
bool Paging::IsUserRangeAccessible(std::uint64_t, std::uint64_t, std::uint64_t, bool) {
NotImplemented("aarch64: Paging::IsUserRangeAccessible not implemented");
void Paging::UnmapUserIn(std::uint64_t root, std::uint64_t virtualAddress) {
pagingLock.Acquire();
volatile Descriptor* entry = WalkExisting(root, virtualAddress);
if (entry != nullptr && (*entry & DescValid)) {
*entry = 0;
Arch::FlushTlbPage(virtualAddress);
}
pagingLock.Release();
}
void Paging::MapEfiRuntime(const montauk::boot::EfiInfo&) {
NotImplemented("aarch64: Paging::MapEfiRuntime not implemented");
void Paging::FreeUserHalf(std::uint64_t root) {
// User space is the lower 128 TiB: level-0 entries 0-255.
for (int i0 = 0; i0 < 256; i0++) {
Descriptor d0 = TableAt(root)[i0];
if ((d0 & DescValid) == 0) continue;
uint64_t l1 = d0 & DescAddrMask;
for (int i1 = 0; i1 < 512; i1++) {
Descriptor d1 = TableAt(l1)[i1];
if ((d1 & DescValid) == 0) continue;
uint64_t l2 = d1 & DescAddrMask;
for (int i2 = 0; i2 < 512; i2++) {
Descriptor d2 = TableAt(l2)[i2];
if ((d2 & DescValid) == 0) continue;
uint64_t l3 = d2 & DescAddrMask;
for (int i3 = 0; i3 < 512; i3++) {
Descriptor leaf = TableAt(l3)[i3];
if ((leaf & DescValid) == 0) continue;
// Write-combining/device pages are not the process's.
if (((leaf & DescAttrMask) >> DescAttrShift) != AttrNormal) continue;
uint64_t pagePhys = leaf & DescAddrMask;
if (pagePhys != 0) {
Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
}
}
Memory::g_pfa->Free((void*)Memory::HHDM(l3));
}
Memory::g_pfa->Free((void*)Memory::HHDM(l2));
}
Memory::g_pfa->Free((void*)Memory::HHDM(l1));
}
}
bool Paging::IsUserRangeAccessible(std::uint64_t root, std::uint64_t virtualAddress,
std::uint64_t size, bool requireWrite) {
if (size == 0) return true;
if (virtualAddress >= UserSpaceEnd) return false;
if (virtualAddress + size < virtualAddress) return false;
if (virtualAddress + size > UserSpaceEnd) return false;
uint64_t page = virtualAddress & ~0xFFFULL;
uint64_t lastPage = (virtualAddress + size - 1) & ~0xFFFULL;
for (;;) {
volatile Descriptor* entry = WalkExisting(root, page);
if (entry == nullptr) return false;
Descriptor desc = *entry;
if (!IsUserLeaf(desc)) return false;
if (requireWrite && (desc & DescApReadOnly)) return false;
if (page == lastPage) break;
page += 0x1000;
}
return true;
}
}
kernel/src/Arch/aarch64/Platform.cpp
+23 -3
View File
@@ -12,12 +12,32 @@
using namespace Kt;
extern "C" char ArchExceptionVectors[]; // Vectors.S
namespace Arch {
// SCTLR_EL1.WXN: writable memory is execute-never
static constexpr uint64_t SctlrWxn = 1ULL << 19;
void EarlyInitialize() {
// No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
asm volatile("msr tpidr_el1, xzr");
// Run on SP_EL1 so exceptions taken from the kernel arrive on the
// stack it is already using (Vectors.S has no SP_EL0 handlers).
asm volatile("mov x9, sp\n\tmsr spsel, #1\n\tmov sp, x9" ::: "x9", "memory");
// The vectors need no allocation, so take over from the
// bootloader's right away.
asm volatile("msr vbar_el1, %0\n\tisb" :: "r"(ArchExceptionVectors) : "memory");
// Kernel mappings are RWX (Paging.cpp), so writable must not
// imply execute-never.
uint64_t sctlr;
asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr));
sctlr &= ~SctlrWxn;
asm volatile("msr sctlr_el1, %0\n\tisb" :: "r"(sctlr));
// CPACR_EL1.FPEN = 0b11: FP/SIMD usable at EL0 and EL1 (the kernel
// itself is built -mgeneral-regs-only; Context.S saves user state).
uint64_t cpacr;
@@ -26,10 +46,10 @@ namespace Arch {
asm volatile("msr cpacr_el1, %0\n\tisb" :: "r"(cpacr));
}
// TODO(aarch64): install the VBAR_EL1 vector table (EL1/EL0 synchronous
// exceptions -> panic / SyscallDispatch, IRQ -> GIC -> ArchIrqDispatch).
// Installed by EarlyInitialize.
void InitializeTraps() {
KernelLogStream(WARNING, "Arch") << "aarch64: exception vectors not implemented";
KernelLogStream(OK, "Arch") << "Installed exception vectors at 0x" << base::hex
<< (uint64_t)ArchExceptionVectors;
}
// MAIR_EL1 attribute indices are chosen together with the page-table
kernel/src/Arch/aarch64/SyscallFrame.hpp
+8 -4
View File
@@ -9,13 +9,17 @@
namespace Arch {
// Saved by the EL0 synchronous-exception vector for `svc #0`:
// x0-x5 carry the arguments and x8 the system-call number.
// The exception frame Vectors.S saves for `svc #0` (the same layout as
// Arch::PanicFrame): x0-x5 carry the arguments and x8 the system-call
// number; the result is written back to x0.
struct SyscallFrame {
uint64_t arg1, arg2, arg3, arg4, arg5, arg6; // x0-x5
uint64_t x6, x7;
uint64_t syscall_nr; // x8
uint64_t elr, spsr, sp_el0; // user PC, PSTATE, SP
uint64_t x19_x30[12]; // callee-saved + LR
uint64_t x9_x30[22];
uint64_t sp_el0, elr, spsr, esr, far;
};
static_assert(sizeof(SyscallFrame) == 288, "must match Vectors.S");
}
kernel/src/Arch/aarch64/Vectors.S
+201
View File
@@ -0,0 +1,201 @@
/*
Vectors.S
aarch64 exception vector table (VBAR_EL1)
Copyright (c) 2026 Daniel Hammer
*/
/*
* Every exception saves the same 288-byte frame on SP_EL1, laid out as
* Arch::PanicFrame (InterruptFrame.hpp) and Arch::SyscallFrame:
*
* 0 x0 .. x30 (x30 at 240)
* 248 interrupted SP (SP_EL0 from EL0, the pre-exception SP_EL1 from EL1)
* 256 ELR_EL1
* 264 SPSR_EL1
* 272 ESR_EL1
* 280 FAR_EL1
*
* The kernel runs on SP_EL1 (SPSel = 1, see Platform.cpp). From EL0,
* SP_EL1 is the top of the thread's kernel stack: ArchEnterUserMode sets
* it before the first ERET and every return to EL0 restores it.
*/
#define FRAME_SIZE 288
#define FRAME_SP 248
#define FRAME_ELR 256
#define FRAME_SPSR 264
#define FRAME_ESR 272
#define FRAME_FAR 280
.text
.macro SAVE_FRAME el
sub sp, sp, #FRAME_SIZE
stp x0, x1, [sp, #0]
stp x2, x3, [sp, #16]
stp x4, x5, [sp, #32]
stp x6, x7, [sp, #48]
stp x8, x9, [sp, #64]
stp x10, x11, [sp, #80]
stp x12, x13, [sp, #96]
stp x14, x15, [sp, #112]
stp x16, x17, [sp, #128]
stp x18, x19, [sp, #144]
stp x20, x21, [sp, #160]
stp x22, x23, [sp, #176]
stp x24, x25, [sp, #192]
stp x26, x27, [sp, #208]
stp x28, x29, [sp, #224]
.if \el == 0
mrs x21, sp_el0
.else
add x21, sp, #FRAME_SIZE
.endif
stp x30, x21, [sp, #240]
mrs x22, elr_el1
mrs x23, spsr_el1
stp x22, x23, [sp, #FRAME_ELR]
mrs x24, esr_el1
mrs x25, far_el1
stp x24, x25, [sp, #FRAME_ESR]
.endm
/* Interrupts are masked again before ELR/SPSR are reloaded: an IRQ taken
in between would overwrite them. */
.macro RESTORE_FRAME el
msr daifset, #2
ldp x22, x23, [sp, #FRAME_ELR]
msr elr_el1, x22
msr spsr_el1, x23
.if \el == 0
ldr x21, [sp, #FRAME_SP]
msr sp_el0, x21
.endif
ldp x0, x1, [sp, #0]
ldp x2, x3, [sp, #16]
ldp x4, x5, [sp, #32]
ldp x6, x7, [sp, #48]
ldp x8, x9, [sp, #64]
ldp x10, x11, [sp, #80]
ldp x12, x13, [sp, #96]
ldp x14, x15, [sp, #112]
ldp x16, x17, [sp, #128]
ldp x18, x19, [sp, #144]
ldp x20, x21, [sp, #160]
ldp x22, x23, [sp, #176]
ldp x24, x25, [sp, #192]
ldp x26, x27, [sp, #208]
ldp x28, x29, [sp, #224]
ldr x30, [sp, #240]
add sp, sp, #FRAME_SIZE
eret
.endm
/* Each vector slot is 128 bytes; the handlers live outside the table. */
.macro VECTOR target
.balign 0x80
b \target
.endm
.balign 0x800
.global ArchExceptionVectors
ArchExceptionVectors:
/* Current EL with SP_EL0: the kernel never runs on SP_EL0. */
VECTOR el1t_unexpected
VECTOR el1t_unexpected
VECTOR el1t_unexpected
VECTOR el1t_unexpected
/* Current EL with SP_ELx */
VECTOR el1h_sync
VECTOR el1h_irq
VECTOR el1h_fiq
VECTOR el1h_serror
/* Lower EL, AArch64 */
VECTOR el0_sync
VECTOR el0_irq
VECTOR el0_fiq
VECTOR el0_serror
/* Lower EL, AArch32: not supported */
VECTOR el0_aarch32
VECTOR el0_aarch32
VECTOR el0_aarch32
VECTOR el0_aarch32
/* ---- Exceptions from the kernel ---- */
el1t_unexpected:
SAVE_FRAME 1
mov x0, sp
mov x1, #0
bl ArchUnexpectedException
el1h_sync:
SAVE_FRAME 1
mov x0, sp
mov x1, #0
bl ArchSyncException
RESTORE_FRAME 1
el1h_irq:
SAVE_FRAME 1
mov x0, sp
mov x1, #0
bl ArchIrqException
RESTORE_FRAME 1
el1h_fiq:
SAVE_FRAME 1
mov x0, sp
mov x1, #1
bl ArchUnexpectedException
el1h_serror:
SAVE_FRAME 1
mov x0, sp
mov x1, #2
bl ArchUnexpectedException
/* ---- Exceptions from user mode ---- */
el0_sync:
SAVE_FRAME 0
lsr x9, x24, #26 /* ESR_EL1.EC */
cmp x9, #0x15 /* SVC from AArch64 */
b.ne 1f
msr daifclr, #2
mov x0, sp
bl SyscallDispatch
str x0, [sp, #0]
bl SchedExitIfKilled
RESTORE_FRAME 0
1: mov x0, sp
mov x1, #1
bl ArchSyncException
RESTORE_FRAME 0
el0_irq:
SAVE_FRAME 0
mov x0, sp
mov x1, #1
bl ArchIrqException
RESTORE_FRAME 0
el0_fiq:
SAVE_FRAME 0
mov x0, sp
mov x1, #1
bl ArchUnexpectedException
el0_serror:
SAVE_FRAME 0
mov x0, sp
mov x1, #2
bl ArchUnexpectedException
el0_aarch32:
SAVE_FRAME 0
mov x0, sp
mov x1, #3
bl ArchUnexpectedException
kernel/src/Boot/Main.cpp
+3
View File
@@ -27,6 +27,7 @@
#include <Pci/Pci.hpp>
#include <Timekeeping/Timer.hpp>
#include <Drivers/Init.hpp>
#include <Drivers/Serial/Uart.hpp>
#include <Graphics/Framebuffer.hpp>
#include <Arch/Cpu.hpp>
#include <Arch/Platform.hpp>
@@ -111,6 +112,8 @@ extern "C" void kmain() {
Hal::ACPI g_acpi((Hal::ACPI::XSDP*)Memory::HHDM(boot.rsdpPhysical));
Drivers::Serial::InitializeConsole(g_acpi.GetXSDT());
if (g_acpi.GetXSDT() != nullptr) {
Hal::AcpiShutdown::Initialize(g_acpi.GetXSDT());
kernel/src/Drivers/Serial/Uart.cpp
+181
View File
@@ -0,0 +1,181 @@
/*
* Uart.cpp
* Polled serial console from the ACPI SPCR table
* Copyright (c) 2026 Daniel Hammer
*/
#include "Uart.hpp"
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Paging.hpp>
#include <Io/IoPort.hpp>
#include <Arch/Cpu.hpp>
using namespace Kt;
namespace Drivers::Serial {
struct GenericAddress {
uint8_t AddressSpaceId; // 0 = system memory, 1 = system I/O
uint8_t RegisterBitWidth;
uint8_t RegisterBitOffset;
uint8_t AccessSize; // 1 = byte, 2 = word, 3 = dword, 4 = qword
uint64_t Address;
} __attribute__((packed));
struct Spcr {
Hal::ACPI::CommonSDTHeader Header;
uint8_t InterfaceType;
uint8_t Reserved[3];
GenericAddress BaseAddress;
uint8_t InterruptType;
uint8_t Irq;
uint32_t GlobalSystemInterrupt;
uint8_t BaudRate;
uint8_t Parity;
uint8_t StopBits;
uint8_t FlowControl;
uint8_t TerminalType;
} __attribute__((packed));
// SPCR / DBG2 serial interface subtypes
static constexpr uint8_t Type16550 = 0x00;
static constexpr uint8_t Type16450 = 0x01;
static constexpr uint8_t TypePl011 = 0x03;
static constexpr uint8_t TypeSbsa32 = 0x0D;
static constexpr uint8_t TypeSbsaGeneric = 0x0E;
static constexpr uint8_t TypeBcm2835 = 0x10;
static constexpr uint8_t Type16550Generic = 0x12;
// PL011: data register, flag register (TX FIFO full)
static constexpr uint64_t Pl011Dr = 0x00;
static constexpr uint64_t Pl011Fr = 0x18;
static constexpr uint32_t Pl011FrTxff = 1u << 5;
// 16550: transmit holding register, line status register (THR empty)
static constexpr uint64_t Uart16550Thr = 0;
static constexpr uint64_t Uart16550Lsr = 5;
static constexpr uint8_t Uart16550LsrThre = 1u << 5;
// Bounded so a wedged or absent UART cannot hang the console.
static constexpr int TxSpinLimit = 100000;
static volatile uint8_t* g_mmio = nullptr;
static uint16_t g_port = 0;
static uint32_t g_stride = 1;
static inline uint32_t Read32(uint64_t offset) {
return *(volatile uint32_t*)(g_mmio + offset);
}
static inline void Write32(uint64_t offset, uint32_t value) {
*(volatile uint32_t*)(g_mmio + offset) = value;
}
static void Pl011Putchar(char c) {
for (int i = 0; i < TxSpinLimit && (Read32(Pl011Fr) & Pl011FrTxff); i++) {
Arch::Pause();
}
Write32(Pl011Dr, (uint8_t)c);
}
static uint8_t Mmio16550Read(uint64_t reg) {
if (g_stride == 4) return (uint8_t)Read32(reg * 4);
return g_mmio[reg];
}
static void Mmio16550Write(uint64_t reg, uint8_t value) {
if (g_stride == 4) Write32(reg * 4, value);
else g_mmio[reg] = value;
}
static void Mmio16550Putchar(char c) {
for (int i = 0; i < TxSpinLimit && !(Mmio16550Read(Uart16550Lsr) & Uart16550LsrThre); i++) {
Arch::Pause();
}
Mmio16550Write(Uart16550Thr, (uint8_t)c);
}
static void Port16550Putchar(char c) {
for (int i = 0; i < TxSpinLimit && !(Io::In8(g_port + Uart16550Lsr) & Uart16550LsrThre); i++) {
Arch::Pause();
}
Io::Out8((uint8_t)c, g_port + Uart16550Thr);
}
static Spcr* FindSpcr(Hal::ACPI::CommonSDTHeader* xsdt) {
uint32_t entryCount = (xsdt->Length - sizeof(Hal::ACPI::CommonSDTHeader)) / 8;
uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(Hal::ACPI::CommonSDTHeader));
for (uint32_t i = 0; i < entryCount; i++) {
auto* header = (Hal::ACPI::CommonSDTHeader*)Memory::HHDM(entries[i]);
const char* sig = header->Signature;
if (sig[0] == 'S' && sig[1] == 'P' && sig[2] == 'C' && sig[3] == 'R') {
return (Spcr*)header;
}
}
return nullptr;
}
static volatile uint8_t* MapRegisters(uint64_t phys) {
uint64_t page = phys & ~0xFFFULL;
Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
return (volatile uint8_t*)Memory::HHDM(phys);
}
void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt) {
if (xsdt == nullptr) return;
Spcr* spcr = FindSpcr(xsdt);
if (spcr == nullptr || spcr->Header.Length < sizeof(Spcr)) {
return;
}
const GenericAddress& reg = spcr->BaseAddress;
if (reg.Address == 0) return;
SerialSink sink = nullptr;
const char* name = "";
bool isIo = reg.AddressSpaceId == 1;
switch (spcr->InterfaceType) {
case TypePl011:
case TypeSbsa32:
case TypeSbsaGeneric:
if (isIo) return;
g_mmio = MapRegisters(reg.Address);
sink = Pl011Putchar;
name = "PL011";
break;
case Type16550:
case Type16450:
case Type16550Generic:
case TypeBcm2835:
if (isIo) {
if (!Io::HasPortIo) return;
g_port = (uint16_t)reg.Address;
sink = Port16550Putchar;
} else {
// The BCM2835 mini UART is a 16550 with 32-bit registers.
g_stride = (reg.RegisterBitWidth == 32 || reg.AccessSize == 3
|| spcr->InterfaceType == TypeBcm2835) ? 4 : 1;
g_mmio = MapRegisters(reg.Address);
sink = Mmio16550Putchar;
}
name = spcr->InterfaceType == TypeBcm2835 ? "BCM2835 mini UART" : "16550";
break;
default:
KernelLogStream(WARNING, "Serial") << "SPCR interface type 0x" << base::hex
<< (uint64_t)spcr->InterfaceType << " not supported";
return;
}
AttachSerialConsole(sink);
KernelLogStream(OK, "Serial") << "Console on " << name << " at " << (isIo ? "port " : "")
<< "0x" << base::hex << reg.Address;
}
}
kernel/src/Drivers/Serial/Uart.hpp
+18
View File
@@ -0,0 +1,18 @@
/*
* Uart.hpp
* Polled serial console from the ACPI SPCR table
* Copyright (c) 2026 Daniel Hammer
*/
#pragma once
#include <ACPI/ACPI.hpp>
namespace Drivers::Serial {
// Find the firmware's serial console (SPCR: PL011, SBSA generic UART,
// 16550 or the BCM2835 mini UART) and mirror the kernel console to it.
// Uses the baud rate the firmware configured. Needs the kernel page
// tables (the registers are mapped uncached through the HHDM).
void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt);
}
kernel/src/Pci/Pci.cpp
+25
View File
@@ -187,7 +187,20 @@ namespace Pci {
Io::Out32(value, ConfigDataPort);
}
// Drivers reach config space through the Legacy* calls below. Without
// an I/O address space (ARM) there is no CF8/CFC mechanism, so they use
// ECAM there, at the access's own width (16/8-bit writes must not
// read-modify-write a neighbouring RW1C register).
static bool EcamCovers(uint8_t bus);
static bool UseEcamForLegacy(uint8_t bus) {
return !Io::HasPortIo && EcamCovers(bus);
}
uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
if (UseEcamForLegacy(bus)) {
return EcamRead32(g_ecamBase, bus, device, function, offset & 0xFC);
}
g_legacyConfigLock.Acquire();
uint32_t value = LegacyRead32Unlocked(bus, device, function, offset);
g_legacyConfigLock.Release();
@@ -205,12 +218,20 @@ namespace Pci {
}
void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) {
if (UseEcamForLegacy(bus)) {
EcamWrite32(g_ecamBase, bus, device, function, offset & 0xFC, value);
return;
}
g_legacyConfigLock.Acquire();
LegacyWrite32Unlocked(bus, device, function, offset, value);
g_legacyConfigLock.Release();
}
void LegacyWrite16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint16_t value) {
if (UseEcamForLegacy(bus)) {
EcamWrite16(g_ecamBase, bus, device, function, offset & 0xFE, value);
return;
}
g_legacyConfigLock.Acquire();
uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
int shift = (offset & 2) * 8;
@@ -221,6 +242,10 @@ namespace Pci {
}
void LegacyWrite8(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint8_t value) {
if (UseEcamForLegacy(bus)) {
EcamWrite8(g_ecamBase, bus, device, function, offset, value);
return;
}
g_legacyConfigLock.Acquire();
uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
int shift = (offset & 3) * 8;
kernel/src/Terminal/Terminal.cpp
+25 -3
View File
@@ -37,6 +37,14 @@ namespace Kt {
// ANSI escape sequence filter state
static bool g_ansiEscape = false;
static SerialSink g_serialSink = nullptr;
static void SerialPutchar(char c) {
if (g_serialSink == nullptr) return;
if (c == '\n') g_serialSink('\r');
g_serialSink(c);
}
static void RingBufferAppend(char c) {
// Strip ANSI escape sequences
if (c == '\033') {
@@ -209,18 +217,20 @@ namespace Kt {
// framebuffer terminal below; storing it in the ring makes file
// consumers treat one logical newline as two line breaks.
RingBufferAppend(c);
SerialPutchar(c);
if (g_suppressKernelLog) {
return;
}
}
} else {
// Once a graphical app takes over, suppress ALL flanterm writes
// (SYS_PRINT from user processes, etc.) to avoid painting text
// over the GUI framebuffer.
if (g_suppressKernelLog && g_kernelLogDepth == 0) {
if (g_suppressKernelLog) {
return;
}
SerialPutchar(c);
}
if (c == '\n') {
flanterm_write(ctx, "\r\n", 2);
@@ -235,6 +245,18 @@ namespace Kt {
}
}
void AttachSerialConsole(SerialSink sink) {
g_termLock.Acquire();
uint64_t start = g_klogCount < KLOG_BUF_SIZE ? 0 : g_klogHead;
for (uint64_t i = 0; i < g_klogCount; i++) {
char c = g_klogBuf[(start + i) % KLOG_BUF_SIZE];
if (c == '\n') sink('\r');
sink(c);
}
g_serialSink = sink;
g_termLock.Release();
}
void SuppressKernelLog() {
g_suppressKernelLog = true;
}
kernel/src/Terminal/Terminal.hpp
+6
View File
@@ -43,6 +43,12 @@ namespace Kt
void Putchar(char c);
void Print(const char *text);
// Mirror console output to a polled serial port (Drivers/Serial). The
// sink receives the whole kernel log, also once the desktop suppresses
// it on screen; what was logged before attaching is replayed first.
using SerialSink = void (*)(char c);
void AttachSerialConsole(SerialSink sink);
// Terminal mutex - protects flanterm from concurrent CPU access.
// Uses Mutex (not Spinlock) to keep interrupts enabled while held,
// preventing dropped PS/2 mouse/keyboard bytes during log output.
scripts/check-arch-neutral.sh
+4 -2
View File
@@ -93,8 +93,10 @@ if [ -n "$missing" ]; then
fi
echo "check-arch-neutral: $ARCH objects compile and every symbol resolves"
if command -v "${KERNEL_LD:-ld.lld}" >/dev/null 2>&1; then
make ARCH="$ARCH" >/dev/null
# Same lookup as kernel/GNUmakefile: a system ld.lld, else rustup's.
lld="${KERNEL_LD:-$(command -v ld.lld || ls "$HOME"/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld 2>/dev/null | head -n 1)}"
if [ -n "$lld" ] && command -v "$lld" >/dev/null 2>&1; then
make ARCH="$ARCH" KERNEL_LD="$lld" >/dev/null
echo "check-arch-neutral: linked kernel/bin-$ARCH/kernel"
fi