feat: aarch64 port - fix build system obstacles, add exception handling, page tables

GNUmakefile

@@ -10,6 +10,22 @@ QEMUFLAGS := -smp 4 -m 2G -d int -D qemu.log -no-reboot
 
 override IMAGE_NAME := montauk-$(ARCH)
 
+# Userspace exists only for x86_64 so far: other architectures get a
+# ramdisk holding just the boot files.
+ifeq ($(ARCH),x86_64)
+    KERNEL_ONLY ?= 0
+else
+    KERNEL_ONLY ?= 1
+endif
+
+override EFI_LOADER_x86_64 := BOOTX64.EFI
+override EFI_LOADER_aarch64 := BOOTAA64.EFI
+override EFI_LOADER_riscv64 := BOOTRISCV64.EFI
+override EFI_LOADER_loongarch64 := BOOTLOONGARCH64.EFI
+
+# AArch64 UEFI firmware for QEMU (Debian/Ubuntu: qemu-efi-aarch64).
+AARCH64_EFI_FIRMWARE ?= /usr/share/qemu-efi-aarch64/QEMU_EFI.fd
+
 # Toolchain for building the 'limine' executable for the host.
 HOST_CC := cc
 HOST_CFLAGS := -g -O2 -pipe
@@ -51,29 +67,34 @@ run-hdd-x86_64: $(IMAGE_NAME).hdd
 		$(QEMUFLAGS)
 
 .PHONY: run-aarch64
+# The machine mirrors the Raspberry Pi 4/400: Cortex-A72 cores and a GICv2.
+# The kernel console is mirrored to the PL011 (-serial stdio).
 run-aarch64: $(IMAGE_NAME).iso
 	qemu-system-$(ARCH) \
-		-M virt \
+		-M virt,gic-version=2 \
 		-cpu cortex-a72 \
 		-device ramfb \
 		-device qemu-xhci,id=xhci \
 		-device usb-kbd,bus=xhci.0 \
 		-device usb-mouse,bus=xhci.0 \
-		-bios /usr/share/ovmf/OVMF.fd \
+		-bios $(AARCH64_EFI_FIRMWARE) \
 		-cdrom $(IMAGE_NAME).iso \
+		-serial stdio \
 		$(QEMUFLAGS)
 
 .PHONY: run-hdd-aarch64
 run-hdd-aarch64: $(IMAGE_NAME).hdd
 	qemu-system-$(ARCH) \
-		-M virt \
+		-M virt,gic-version=2 \
 		-cpu cortex-a72 \
 		-device ramfb \
 		-device qemu-xhci,id=xhci \
 		-device usb-kbd,bus=xhci.0 \
 		-device usb-mouse,bus=xhci.0 \
-		-bios /usr/share/ovmf/OVMF.fd \
+		-bios $(AARCH64_EFI_FIRMWARE) \
-		-hda $(IMAGE_NAME).hdd \
+		-drive if=none,id=hd0,format=raw,file=$(IMAGE_NAME).hdd \
+		-device virtio-blk-pci,drive=hd0 \
+		-serial stdio \
 		$(QEMUFLAGS)
 
 .PHONY: run-riscv64
@@ -196,13 +217,23 @@ ports: programs
 	fi
 
 .PHONY: ramdisk
+ifeq ($(KERNEL_ONLY),1)
+ramdisk: limine/limine kernel
+	rm -rf .kernel-only-root
+	mkdir -p .kernel-only-root/boot/limine
+	cp -v limine/$(EFI_LOADER_$(ARCH)) limine.conf .kernel-only-root/boot/limine/
+	cp -v kernel/bin-$(ARCH)/kernel .kernel-only-root/boot/
+	./scripts/mkramdisk.sh .kernel-only-root ramdisk.tar
+	rm -rf .kernel-only-root
+else
 ramdisk: limine/limine kernel programs
 	mkdir -p programs/bin/boot/limine
-	cp -v limine/BOOTX64.EFI programs/bin/boot/limine/
+	cp -v limine/$(EFI_LOADER_$(ARCH)) programs/bin/boot/limine/
 	cp -v limine.conf programs/bin/boot/limine/
 	cp -v kernel/bin-$(ARCH)/kernel programs/bin/boot/
 	rm -f programs/bin/boot/ramdisk.tar
 	./scripts/mkramdisk.sh programs/bin ramdisk.tar
+endif
 
 $(IMAGE_NAME).iso: limine/limine kernel ramdisk
 	rm -rf iso_root

docs/porting.md

@@ -36,7 +36,7 @@ kernel/src/Arch/
   Platform.hpp      boot-sequence hooks, platform device list, reboot
   CpuPower.hpp      frequency scaling / thermal governor (may be no-ops)
   x86_64/           the complete port
-  aarch64/          skeleton port
+  aarch64/          the port in progress (see below)
 ```
 
 Other interfaces a port implements:
@@ -85,25 +85,48 @@ are `Cpu.hpp`, `IoPort.hpp`, `PerCpu.hpp`, `Context.hpp`, `Abi.hpp` and
 - Interrupts are numbered IRQ slots (`Arch::IRQ_COUNT`); slot numbers for
   MSI-capable drivers are fixed in the drivers.
 
-## The aarch64 skeleton
+## The aarch64 port
 
 Real: CPU primitives (DAIF, WFI, `CNTVCT_EL0`, TLBI, TTBR0), per-CPU data via
 `TPIDR_EL1`, the context switch and EL0 entry (`Context.S`), TLS through
 `TPIDR_EL0`, the ELF/TLS/exit-stub ABI (syscall number in x8), the IRQ slot
-dispatch table (`ArchIrqDispatch`), panic-frame printing.
+dispatch table (`ArchIrqDispatch`), panic-frame printing, and:
+
+- Exception vectors (`Vectors.S`, installed in `EarlyInitialize`). Every
+  exception saves one 288-byte frame that is both `Arch::PanicFrame` and
+  `Arch::SyscallFrame`. EL0 `svc` goes to `SyscallDispatch`; other
+  synchronous exceptions go to `ArchSyncException` (`Exceptions.cpp`), which
+  grows user stacks and lazy anonymous memory, kills faulting processes with
+  a crash report, and panics on kernel faults.
+- Translation tables (`Paging.cpp`): 4 KiB granule, 48-bit VAs. The kernel
+  half lives in its own TTBR1 table; `KernelRoot()` is the TTBR0 table of
+  kernel contexts (EFI identity mappings only, so null pointers fault). The
+  HHDM maps only the memory map's RAM and firmware regions, never the holes
+  between them (device space on ARM). MAIR: 0 Normal WB, 1 Normal NC
+  (`MapWC`), 2 Device-nGnRE (`MapMMIO`).
 
 Not yet written (each is marked `TODO(aarch64)`):
 
-- `Paging.cpp`: translation tables and MAIR (every method panics).
-- Exception vectors (`VBAR_EL1`): synchronous faults, `svc`, IRQs.
 - GIC driver: SPI routing, SGIs for IPIs, MSIs via ITS/GICv2m
-  (`ComposeMsi` returns false, so drivers fall back to INTx/polling).
+  (`ComposeMsi` returns false, so drivers fall back to INTx/polling). IRQs
-- Generic timer tick (`Arch::Timer::InitializeBsp` returns false).
+  reach `ArchIrqException`, which only logs them.
+- Generic timer tick (`Arch::Timer::InitializeBsp` returns false). Boot
+  currently stops in the first driver that waits on `GetMilliseconds()`
+  (xHCI port power-up).
 - Secondary CPU start-up, PSCI reset and suspend.
 
-Building it needs `clang` and `ld.lld` (`make kernel ARCH=aarch64`;
+Building it needs `clang` and `ld.lld`; without a system `ld.lld` the
-`KERNEL_LD=` points at a different `ld.lld`, e.g. rustup's
+kernel makefile uses the one rustup ships
-`lib/rustlib/x86_64-unknown-linux-gnu/bin/gcc-ld/ld.lld`).
+(`~/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld`), and `KERNEL_LD=`
+overrides both. `make run ARCH=aarch64` boots a kernel-only image (userspace
+is x86_64-only so far) on QEMU `virt` with a GICv2 and Cortex-A72 cores, the
+Raspberry Pi 4/400's configuration; it needs `qemu-system-aarch64` and
+`qemu-efi-aarch64`.
+
+The kernel console is mirrored to the serial port the ACPI SPCR table names
+(PL011, SBSA UART, 16550 or the BCM2835 mini UART; `Drivers/Serial/Uart.cpp`),
+including the log written before the port was found. On QEMU that is
+`-serial stdio`.
 
 ## Adding another architecture
 

kernel/GNUmakefile

@@ -291,7 +291,9 @@ cc-runtime-$(ARCH)/cc-runtime.a: GNUmakefile cc-runtime/*
 # so a clang-built non-x86 kernel is linked by ld.lld directly.
 ifneq ($(ARCH),x86_64)
     ifeq ($(CXX_IS_CLANG),1)
-        KERNEL_LD ?= ld.lld
+        # Without a system lld, fall back to the one rustup ships.
+        KERNEL_LD ?= $(firstword $(shell command -v ld.lld) \
+            $(wildcard $(HOME)/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld))
         override LD_EMULATION_aarch64 := aarch64elf
         override LD_EMULATION_riscv64 := elf64lriscv
         override LD_EMULATION_loongarch64 := elf64loongarch

kernel/src/Api/BuildNo.hpp

@@ -12,4 +12,4 @@
 
 #pragma once
 
-#define MONTAUK_BUILD_NUMBER 26
+#define MONTAUK_BUILD_NUMBER 29

kernel/src/Arch/aarch64/Cpu.hpp

@@ -92,9 +92,10 @@ namespace Arch {
         asm volatile("dsb sy\n\tisb" ::: "memory");
     }
 
+    // The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0.
     inline void FlushTlbPage(uint64_t virtualAddress) {
         asm volatile("dsb ishst\n\ttlbi vaae1is, %0\n\tdsb ish\n\tisb"
-                     :: "r"(virtualAddress >> 12) : "memory");
+                     :: "r"((virtualAddress >> 12) & ((1ULL << 44) - 1)) : "memory");
     }
 
     inline void FlushTlbAll() {

kernel/src/Arch/aarch64/Exceptions.cpp

@@ -0,0 +1,184 @@
+/*
+    * Exceptions.cpp
+    * aarch64 synchronous exceptions: faults, user crashes, kernel panics
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include <Arch/aarch64/InterruptFrame.hpp>
+#include <Arch/Cpu.hpp>
+#include <Arch/Smp.hpp>
+#include <Common/Panic.hpp>
+#include <Common/TerminationCodes.hpp>
+#include <CppLib/Stream.hpp>
+#include <Memory/Paging.hpp>
+#include <Sched/Scheduler.hpp>
+#include <Sched/CrashReport.hpp>
+#include <Api/Heap.hpp>
+#include <Timekeeping/Timer.hpp>
+
+using namespace Kt;
+
+namespace {
+
+    // ESR_EL1.EC exception classes
+    constexpr uint64_t EcUnknown         = 0x00;
+    constexpr uint64_t EcIllegalState    = 0x0E;
+    constexpr uint64_t EcInstAbortLower  = 0x20;
+    constexpr uint64_t EcInstAbortSame   = 0x21;
+    constexpr uint64_t EcPcAlignment     = 0x22;
+    constexpr uint64_t EcDataAbortLower  = 0x24;
+    constexpr uint64_t EcDataAbortSame   = 0x25;
+    constexpr uint64_t EcSpAlignment     = 0x26;
+    constexpr uint64_t EcFpException     = 0x2C;
+    constexpr uint64_t EcBreakpointLower = 0x30;
+    constexpr uint64_t EcBrk             = 0x3C;
+
+    constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL;
+
+    uint64_t ExceptionClass(const Arch::PanicFrame* frame) {
+        return (frame->esr >> 26) & 0x3F;
+    }
+
+    bool IsAbort(uint64_t ec) {
+        return ec == EcInstAbortLower || ec == EcInstAbortSame
+            || ec == EcDataAbortLower || ec == EcDataAbortSame;
+    }
+
+    // Fault status code (ISS[5:0]) of an abort: translation faults mean
+    // the page is not mapped, permission faults that it is.
+    bool IsTranslationFault(uint64_t esr) {
+        uint64_t fsc = esr & 0x3F;
+        return fsc >= 0x04 && fsc <= 0x07;
+    }
+
+    bool IsPermissionFault(uint64_t esr) {
+        uint64_t fsc = esr & 0x3F;
+        return fsc >= 0x0C && fsc <= 0x0F;
+    }
+
+    const char* ExceptionName(uint64_t ec) {
+        switch (ec) {
+            case EcUnknown:         return "Undefined Instruction";
+            case EcIllegalState:    return "Illegal Execution State";
+            case EcInstAbortLower:
+            case EcInstAbortSame:   return "Instruction Abort";
+            case EcPcAlignment:     return "PC Alignment Fault";
+            case EcDataAbortLower:
+            case EcDataAbortSame:   return "Data Abort";
+            case EcSpAlignment:     return "SP Alignment Fault";
+            case EcFpException:     return "Floating-Point Exception";
+            case EcBrk:             return "Breakpoint (BRK)";
+            default:
+                if (ec >= EcBreakpointLower && ec <= 0x35) return "Debug Exception";
+                return "Synchronous Exception";
+        }
+    }
+
+    TerminationCodes TerminationFromClass(uint64_t ec) {
+        if (ec == EcUnknown || ec == EcIllegalState) return TerminatedIllegalInstruction;
+        if (ec == EcBrk || (ec >= EcBreakpointLower && ec <= 0x35)) return TerminatedBreakpoint;
+        if (ec == EcFpException) return TerminatedArithmeticError;
+        return TerminatedAccessViolation;
+    }
+
+    template<size_t N>
+    void CopyName(char (&dst)[N], const char* src) {
+        size_t i = 0;
+        for (; i + 1 < N && src[i]; i++) dst[i] = src[i];
+        dst[i] = '\0';
+    }
+
+    // Kill the faulting process with a crash report. Never returns.
+    [[noreturn]] void TerminateUserProcess(Arch::PanicFrame* frame, uint64_t ec) {
+        // Teardown can wait on devices and other CPUs; keep the clock running.
+        Arch::EnableInterrupts();
+
+        auto* proc = Sched::GetCurrentProcessPtr();
+        const char* name = ExceptionName(ec);
+        KernelLogStream(ERROR, "Exception") << name << " in process \"" << proc->name
+            << "\" (pid " << proc->pid << ", ESR 0x" << base::hex << frame->esr
+            << ", PC 0x" << frame->elr << ") - process terminated";
+
+        CrashReport::Report rep{};
+        rep.valid = true;
+        rep.pid = proc->pid;
+        CopyName(rep.processName, proc->name);
+        rep.exceptionVector = (uint8_t)ec;
+        CopyName(rep.exceptionName, name);
+        rep.instructionPointer = frame->elr;
+        rep.flags = frame->spsr;
+        rep.stackPointer = frame->sp;
+
+        constexpr uint64_t snapshotBytes = CrashReport::StackSnapshotWords * sizeof(uint64_t);
+        if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pageTableRoot, frame->sp,
+                                                       snapshotBytes, false)) {
+            auto* userStack = (const volatile uint64_t*)frame->sp;
+            for (int word = 0; word < CrashReport::StackSnapshotWords; word++)
+                rep.stackWords[word] = userStack[word];
+            rep.stackWordCount = CrashReport::StackSnapshotWords;
+        }
+
+        if (IsAbort(ec)) {
+            rep.faultingAddress = frame->far;
+            rep.pfPresent = IsPermissionFault(frame->esr);
+            rep.pfWrite = ec == EcDataAbortLower && (frame->esr & (1ULL << 6)) != 0;
+            rep.pfUser = 1;
+            rep.pfInstructionFetch = ec == EcInstAbortLower;
+        }
+        rep.timestampTick = Timekeeping::GetTicks();
+
+        CrashReport::AddReport(rep);
+        Sched::SpawnCrashPad(proc->pid);
+
+        // As on x86: a sibling thread's fault must end the whole process.
+        int code = TerminationFromClass(ec);
+        if (Sched::GetCurrentThreadPtr() != proc) {
+            Sched::KillProcess(proc->pid, code);
+        } else {
+            Sched::SetProcessExitCode(proc->pid, code);
+        }
+
+        Sched::ExitProcess();
+        __builtin_unreachable();
+    }
+
+    // A not-present user page: anonymous memory mapped lazily or the user
+    // stack growing. Also reached from the kernel touching user buffers.
+    bool TryResolveUserFault(const Arch::PanicFrame* frame, uint64_t ec) {
+        if (!IsAbort(ec) || !IsTranslationFault(frame->esr)) return false;
+        if (frame->far >= UserSpaceEnd) return false;
+
+        auto* cpu = Smp::TryGetCurrentCpuData();
+        if (cpu == nullptr || cpu->currentSlot < 0 || Sched::GetCurrentPid() < 0) return false;
+
+        bool isData = ec == EcDataAbortLower || ec == EcDataAbortSame;
+        Memory::VMM::PageFault fault{false, isData && (frame->esr & (1ULL << 6)) != 0, !isData};
+        return montauk::abi::TryHandleAnonymousPageFault(frame->far, fault)
+            || Sched::TryGrowUserStack(frame->far);
+    }
+
+}
+
+extern "C" void ArchSyncException(Arch::PanicFrame* frame, bool fromUser) {
+    uint64_t ec = ExceptionClass(frame);
+
+    if (TryResolveUserFault(frame, ec)) {
+        return;
+    }
+
+    if (fromUser && Sched::GetCurrentPid() >= 0) {
+        TerminateUserProcess(frame, ec);
+    }
+
+    Panic(ExceptionName(ec), frame);
+}
+
+// FIQs are never routed, SErrors are asynchronous external aborts, and the
+// kernel neither runs on SP_EL0 nor supports AArch32 user code.
+extern "C" void ArchUnexpectedException(Arch::PanicFrame* frame, uint64_t kind) {
+    static const char* const names[] = {
+        "Exception on SP_EL0", "Unexpected FIQ", "SError (asynchronous abort)",
+        "AArch32 exception",
+    };
+    Panic(kind < 4 ? names[kind] : "Unexpected exception", frame);
+}

kernel/src/Arch/aarch64/InterruptFrame.hpp

@@ -9,7 +9,7 @@
 
 namespace Arch {
 
-    // What the exception vectors save before calling into C++.
+    // What the exception vectors (Vectors.S) save before calling into C++.
     struct PanicFrame {
         uint64_t x[31];   // x0-x30
         uint64_t sp;      // interrupted SP (SP_EL0 for EL0, else SP_EL1)
@@ -19,4 +19,6 @@ namespace Arch {
         uint64_t far;     // FAR_EL1: faulting address
     };
 
+    static_assert(sizeof(PanicFrame) == 288, "must match Vectors.S");
+
 }

kernel/src/Arch/aarch64/Interrupts.cpp

@@ -45,6 +45,17 @@ namespace Arch {
 
 }
 
+// IRQ exception vector (Vectors.S).
+// TODO(aarch64): acknowledge through the GIC CPU interface, translate the
+// INTID to a slot and call ArchIrqDispatch.
+extern "C" void ArchIrqException(void*, bool) {
+    static bool reported = false;
+    if (!reported) {
+        reported = true;
+        Kt::KernelLogStream(Kt::WARNING, "IRQ") << "aarch64: IRQ taken with no interrupt controller driver";
+    }
+}
+
 extern "C" void ArchIrqDispatch(uint64_t irq, bool fromUser) {
     if (irq < Arch::IRQ_COUNT && Arch::g_irqHandlers[irq] != nullptr) {
         Arch::g_irqHandlers[irq]((uint8_t)irq, fromUser);

kernel/src/Arch/aarch64/Paging.cpp

@@ -5,29 +5,67 @@
 */
 
 #include <Memory/Paging.hpp>
+#include <Memory/PageFrameAllocator.hpp>
+#include <Memory/HHDM.hpp>
 #include <Arch/Cpu.hpp>
 #include <Common/Panic.hpp>
 #include <CppLib/Spinlock.hpp>
 
-// TODO(aarch64): 4 KiB-granule, 48-bit VA translation tables. The kernel
+// 4 KiB granule, 48-bit virtual addresses, four levels (0-3). The kernel
-// half lives in TTBR1_EL1 and is never switched; "address space roots"
+// half is translated through TTBR1_EL1 (g_kernelHalfRoot), which never
-// handed to the generic kernel are TTBR0_EL1 tables, so
+// changes after Init. The "address space roots" the generic kernel switches
-// CreateUserAddressSpace has no kernel half to copy. MAIR_EL1 needs
+// between are TTBR0_EL1 tables: a process's root holds only its own lower
-// Normal-WB, Normal-NC (MapWC) and Device-nGnRE (MapMMIO) indices.
+// half, and KernelRoot() is the TTBR0 table of kernel contexts, empty apart
+// from the EFI runtime identity mappings (so null pointers fault).
+//
+// The HHDM covers only what the memory map lists as RAM or firmware data:
+// unlike on x86, the holes between are device space on ARM machines, and a
+// cacheable mapping of device registers invites speculative accesses.
 
 namespace Memory::VMM {
 
     Paging* g_paging = nullptr;
 
+    static std::uint64_t g_hhdmLimit = 0;
+
+    static std::uint64_t g_kernelHalfRoot = 0;
+
+    std::uint64_t Paging::HhdmLimit() { return g_hhdmLimit; }
+
+    // Translation table descriptor bits (VMSAv8-64, stage 1)
+    static constexpr uint64_t DescValid      = 1ULL << 0;
+    static constexpr uint64_t DescTable      = 1ULL << 1;   // table (levels 0-2) or page (level 3)
+    static constexpr uint64_t DescAttrShift  = 2;           // AttrIndx[2:0], into MAIR_EL1
+    static constexpr uint64_t DescAttrMask   = 7ULL << DescAttrShift;
+    static constexpr uint64_t DescApUser     = 1ULL << 6;   // AP[1]: EL0 may access
+    static constexpr uint64_t DescApReadOnly = 1ULL << 7;   // AP[2]
+    static constexpr uint64_t DescShInner    = 3ULL << 8;
+    static constexpr uint64_t DescAf         = 1ULL << 10;  // access flag (no AF faults)
+    static constexpr uint64_t DescNotGlobal  = 1ULL << 11;
+    static constexpr uint64_t DescPxn        = 1ULL << 53;
+    static constexpr uint64_t DescUxn        = 1ULL << 54;
+    static constexpr uint64_t DescAddrMask   = 0x0000FFFFFFFFF000ULL;
+
+    // MAIR_EL1 attribute indices
+    static constexpr uint64_t AttrNormal   = 0;  // write-back, read/write-allocate
+    static constexpr uint64_t AttrNormalNc = 1;  // non-cacheable (write-combining)
+    static constexpr uint64_t AttrDevice   = 2;  // Device-nGnRE
+    static constexpr uint64_t MairValue = 0xFFULL << (8 * AttrNormal)
+                                        | 0x44ULL << (8 * AttrNormalNc)
+                                        | 0x04ULL << (8 * AttrDevice);
+
+    static constexpr uint64_t KernelHalfBase = 0xFFFF000000000000ULL;
+    static constexpr uint64_t UserSpaceEnd   = 0x0000800000000000ULL;
+
+    // EFI memory types whose runtime regions are device registers
+    static constexpr uint32_t EfiMemoryMappedIo          = 11;
+    static constexpr uint32_t EfiMemoryMappedIoPortSpace = 12;
+
     static kcp::Mutex pagingLock;
 
     void LockUserPaging()   { pagingLock.Acquire(); }
     void UnlockUserPaging() { pagingLock.Release(); }
 
-    [[noreturn]] static void NotImplemented(const char* what) {
-        Panic(what, nullptr);
-    }
-
     void ActivateKernelAddressSpace() {
         Arch::WritePageTableRoot(g_paging->KernelRoot());
     }
@@ -36,75 +74,431 @@ namespace Memory::VMM {
         Arch::FlushTlbAll();
     }
 
+    using Descriptor = uint64_t;
+
+    static inline volatile Descriptor* TableAt(uint64_t phys) {
+        return (volatile Descriptor*)Memory::HHDM(phys);
+    }
+
+    static inline uint64_t Index(uint64_t va, int level) {
+        return (va >> (39 - 9 * level)) & 0x1FF;
+    }
+
+    // Order descriptor writes before the table walker (and later
+    // instructions) can observe them.
+    static inline void PublishTables() {
+        asm volatile("dsb ishst\n\tisb" ::: "memory");
+    }
+
+    static inline uint64_t ReadTtbr1() {
+        uint64_t ttbr;
+        asm volatile("mrs %0, ttbr1_el1" : "=r"(ttbr));
+        return ttbr & Arch::TtbrAddressMask;
+    }
+
+    // The level-3 descriptor for `va`, creating intermediate tables.
+    // Returns nullptr when out of memory or when a block mapping covers
+    // `va` (only the bootloader's tables contain blocks).
+    static volatile Descriptor* WalkCreate(uint64_t root, uint64_t va) {
+        uint64_t table = root;
+        for (int level = 0; level < 3; level++) {
+            volatile Descriptor* entry = &TableAt(table)[Index(va, level)];
+            Descriptor desc = *entry;
+            if ((desc & DescValid) == 0) {
+                void* page = Memory::g_pfa->AllocateZeroed();
+                if (page == nullptr) return nullptr;
+                table = Memory::SubHHDM(page);
+                // The zeroed table must be visible before the walker can
+                // reach it.
+                asm volatile("dsb ishst" ::: "memory");
+                *entry = table | DescTable | DescValid;
+            } else if ((desc & DescTable) == 0) {
+                return nullptr;
+            } else {
+                table = desc & DescAddrMask;
+            }
+        }
+        return &TableAt(table)[Index(va, 3)];
+    }
+
+    // The level-3 descriptor for `va`, or nullptr if a table is missing.
+    static volatile Descriptor* WalkExisting(uint64_t root, uint64_t va) {
+        uint64_t table = root;
+        for (int level = 0; level < 3; level++) {
+            Descriptor desc = TableAt(table)[Index(va, level)];
+            if ((desc & (DescValid | DescTable)) != (DescValid | DescTable)) return nullptr;
+            table = desc & DescAddrMask;
+        }
+        return &TableAt(table)[Index(va, 3)];
+    }
+
+    // Install a leaf. Replacing a live translation (other address or
+    // attributes) needs break-before-make: invalidate, flush, then write.
+    static void SetLeaf(volatile Descriptor* entry, Descriptor desc, uint64_t va) {
+        Descriptor old = *entry;
+        if (old == desc) return;
+        if (old & DescValid) {
+            *entry = 0;
+            Arch::FlushTlbPage(va);
+        }
+        *entry = desc;
+        PublishTables();
+    }
+
+    static Descriptor KernelLeaf(uint64_t phys, uint64_t attr) {
+        Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
+                        | (attr << DescAttrShift) | DescUxn;
+        // Kernel memory stays executable, as on x86; device memory never is.
+        if (attr != AttrNormal) desc |= DescPxn;
+        return desc;
+    }
+
+    static Descriptor UserLeaf(uint64_t phys, uint64_t attr, bool writable, bool executable) {
+        Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
+                        | DescNotGlobal | (attr << DescAttrShift) | DescApUser | DescPxn;
+        if (!writable) desc |= DescApReadOnly;
+        if (!executable) desc |= DescUxn;
+        return desc;
+    }
+
+    static inline bool IsUserLeaf(Descriptor desc) {
+        return (desc & (DescValid | DescTable)) == (DescValid | DescTable) && (desc & DescApUser);
+    }
+
+    // Translate through a table tree that may contain block mappings (the
+    // bootloader's). Returns 0 if unmapped.
+    static uint64_t Translate(uint64_t root, uint64_t va) {
+        uint64_t table = root;
+        for (int level = 0; level <= 3; level++) {
+            Descriptor desc = TableAt(table)[Index(va, level)];
+            if ((desc & DescValid) == 0) return 0;
+
+            uint64_t span = 1ULL << (39 - 9 * level);
+            bool leaf = level == 3 || (desc & DescTable) == 0;
+            if (leaf) {
+                if (level == 0) return 0;  // no level-0 blocks with 4 KiB granules
+                return ((desc & DescAddrMask) & ~(span - 1)) | (va & (span - 1));
+            }
+            table = desc & DescAddrMask;
+        }
+        return 0;
+    }
+
+    void Paging::MapEfiRuntime(const montauk::boot::EfiInfo& efi) {
+        if (efi.memoryMap == nullptr || efi.descriptorSize == 0) return;
+
+        struct EfiMemDesc {
+            uint32_t Type;
+            uint64_t PhysicalStart;
+            uint64_t VirtualStart;
+            uint64_t NumberOfPages;
+            uint64_t Attribute;
+        };
+
+        static constexpr uint64_t EFI_MEMORY_RUNTIME = 0x8000000000000000ULL;
+
+        auto* base = (uint8_t*)efi.memoryMap;
+        uint64_t count = efi.memoryMapSize / efi.descriptorSize;
+
+        for (uint64_t i = 0; i < count; i++) {
+            auto* desc = (EfiMemDesc*)(base + i * efi.descriptorSize);
+            if ((desc->Attribute & EFI_MEMORY_RUNTIME) == 0) continue;
+
+            bool device = desc->Type == EfiMemoryMappedIo || desc->Type == EfiMemoryMappedIoPortSpace;
+            for (uint64_t p = 0; p < desc->NumberOfPages; p++) {
+                uint64_t addr = desc->PhysicalStart + p * 0x1000;
+                // Firmware code runs at its HHDM alias (Efi::Init calls the
+                // services there) and reaches its data at physical addresses.
+                // Runtime regions are reserved memory, outside the HHDM.
+                if (device) {
+                    MapMMIO(addr, addr);
+                    MapMMIO(addr, Memory::HHDM(addr));
+                } else {
+                    Map(addr, addr);
+                    Map(addr, Memory::HHDM(addr));
+                }
+            }
+        }
+
+        FlushTLB();
+    }
+
     Paging::Paging() {
-        NotImplemented("aarch64: virtual memory not implemented");
+        rootPhys = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
+        g_kernelHalfRoot = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
     }
 
-    void Paging::Init(std::uint64_t, std::uint64_t, const montauk::boot::MemoryMap&,
+    // Kernel mappings of lower-half addresses (EFI identity mappings) go to
-                      const montauk::boot::Framebuffer&) {
+    // the kernel contexts' TTBR0 table.
-        NotImplemented("aarch64: Paging::Init not implemented");
+    static uint64_t KernelRootFor(uint64_t kernelContextRoot, uint64_t va) {
+        return va >= KernelHalfBase ? g_kernelHalfRoot : kernelContextRoot;
     }
 
-    std::uint64_t Paging::HhdmLimit() { return 0; }
+    void Paging::Init(std::uint64_t kernelBaseVirt, std::uint64_t kernelSize, const montauk::boot::MemoryMap& memMap,
-
+                      const montauk::boot::Framebuffer& framebuffer) {
-    void Paging::Map(std::uint64_t, std::uint64_t) {
+        if (Memory::HHDMBase < KernelHalfBase) {
-        NotImplemented("aarch64: Paging::Map not implemented");
+            Panic("aarch64: HHDM outside the 48-bit kernel half", nullptr);
         }
 
-    void Paging::MapMMIO(std::uint64_t, std::uint64_t) {
+        uint64_t bootTtbr1 = ReadTtbr1();
-        NotImplemented("aarch64: Paging::MapMMIO not implemented");
+        uint64_t bootTcr, bootMair;
+        asm volatile("mrs %0, tcr_el1" : "=r"(bootTcr));
+        asm volatile("mrs %0, mair_el1" : "=r"(bootMair));
+        Kt::KernelLogStream(Kt::DEBUG, "VMM") << "Bootloader TCR 0x" << base::hex << bootTcr
+            << ", MAIR 0x" << bootMair << ", TTBR1 0x" << bootTtbr1;
+
+        // The kernel image, at the physical pages the bootloader loaded it to.
+        for (uint64_t va = kernelBaseVirt; va < kernelBaseVirt + kernelSize; va += 0x1000) {
+            uint64_t phys = Translate(bootTtbr1, va);
+            if (phys == 0) Panic("aarch64: kernel page missing from the bootloader's tables", nullptr);
+            Map(phys, va);
         }
 
-    void Paging::MapWC(std::uint64_t, std::uint64_t) {
+        // HHDM: every region that is real backing store (see the top of
-        NotImplemented("aarch64: Paging::MapWC not implemented");
+        // this file). The limit is published for the AML interpreter.
+        uint64_t maxPhysAddr = 0;
+        for (size_t i = 0; i < memMap.count; i++) {
+            const auto& region = memMap.regions[i];
+            if (region.kind == montauk::boot::MemoryKind::Reserved
+             || region.kind == montauk::boot::MemoryKind::Unknown) {
+                continue;
             }
 
-    void Paging::MapUser(std::uint64_t, std::uint64_t) {
+            uint64_t start = region.base & ~0xFFFULL;
-        NotImplemented("aarch64: Paging::MapUser not implemented");
+            uint64_t end = (region.base + region.length + 0xFFF) & ~0xFFFULL;
+            bool framebufferRegion = region.kind == montauk::boot::MemoryKind::Framebuffer;
+            for (uint64_t phys = start; phys < end; phys += 0x1000) {
+                if (framebufferRegion) MapWC(phys, Memory::HHDM(phys));
+                else Map(phys, Memory::HHDM(phys));
+            }
+            if (end > maxPhysAddr) maxPhysAddr = end;
+        }
+        g_hhdmLimit = maxPhysAddr;
+
+        // The framebuffer may not be listed in the memory map at all.
+        if (framebuffer.address != 0) {
+            uint64_t fbPhys = Memory::SubHHDM(framebuffer.address) & ~0xFFFULL;
+            uint64_t fbPages = (framebuffer.height * framebuffer.pitch + 0xFFF) / 0x1000;
+            for (uint64_t p = 0; p < fbPages; p++) {
+                uint64_t phys = fbPhys + p * 0x1000;
+                MapWC(phys, Memory::HHDM(phys));
+            }
         }
 
-    std::uint64_t Paging::GetPhysAddr(std::uint64_t, std::uint64_t) {
+        // 48-bit VAs in both halves, 4 KiB granules, write-back walks of
-        NotImplemented("aarch64: Paging::GetPhysAddr not implemented");
+        // inner-shareable tables; the physical size is what the CPU has.
+        uint64_t mmfr0;
+        asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0));
+        uint64_t ips = mmfr0 & 0xF;
+        if (ips > 5) ips = 5;  // 52-bit output needs LPA2 descriptors
+        uint64_t tcr = 16ULL                          // T0SZ
+                     | 1ULL << 8 | 1ULL << 10        // IRGN0, ORGN0: write-back
+                     | 3ULL << 12                    // SH0: inner shareable
+                     | 0ULL << 14                    // TG0: 4 KiB
+                     | 16ULL << 16                   // T1SZ
+                     | 1ULL << 24 | 1ULL << 26       // IRGN1, ORGN1
+                     | 3ULL << 28                    // SH1
+                     | 2ULL << 30                    // TG1: 4 KiB
+                     | ips << 32;                    // IPS
+
+        // The new tables map the code running here at the same address, so
+        // the switch is a register write followed by a full invalidate.
+        asm volatile(
+            "dsb ish\n\t"
+            "msr mair_el1, %0\n\t"
+            "msr tcr_el1, %1\n\t"
+            "isb\n\t"
+            "msr ttbr1_el1, %2\n\t"
+            "msr ttbr0_el1, %3\n\t"
+            "isb\n\t"
+            "tlbi vmalle1is\n\t"
+            "dsb ish\n\t"
+            "isb"
+            :: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(rootPhys) : "memory");
+
+        Kt::KernelLogStream(Kt::OK, "VMM") << "Switched to kernel translation tables (HHDM limit 0x"
+            << base::hex << g_hhdmLimit << ")";
+    }
+
+    static void MapKernelPage(uint64_t root, uint64_t phys, uint64_t va, uint64_t attr, const char* what) {
+        if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
+            Panic(what, nullptr);
+        }
+        volatile Descriptor* entry = WalkCreate(KernelRootFor(root, va), va);
+        if (entry == nullptr) {
+            Panic("OOM in Paging (kernel page table allocation)", nullptr);
+        }
+        SetLeaf(entry, KernelLeaf(phys, attr), va);
+    }
+
+    void Paging::Map(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormal,
+                      "Value that isn't page-aligned passed as address to Paging::Map!");
+    }
+
+    void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrDevice,
+                      "Value that isn't page-aligned passed as address to Paging::MapMMIO!");
+    }
+
+    void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormalNc,
+                      "Value that isn't page-aligned passed as address to Paging::MapWC!");
+    }
+
+    void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
+            Panic("Value that isn't page-aligned passed as address to Paging::MapUser!", nullptr);
+        }
+        volatile Descriptor* entry = WalkCreate(rootPhys, virtualAddress);
+        if (entry == nullptr) {
+            Panic("OOM in Paging::MapUser (page table allocation)", nullptr);
+        }
+        SetLeaf(entry, UserLeaf(physicalAddress, AttrNormal, true, true), virtualAddress);
+    }
+
+    std::uint64_t Paging::GetPhysAddr(std::uint64_t root, std::uint64_t virtualAddress) {
+        if (virtualAddress >= KernelHalfBase) {
+            root = g_kernelHalfRoot;
+        }
+        return Translate(root, virtualAddress);
     }
 
     std::uint64_t Paging::GetPhysAddr(std::uint64_t virtualAddress) {
         return GetPhysAddr(rootPhys, virtualAddress);
     }
 
+    // The kernel half lives in TTBR1, so a new address space starts empty.
     std::uint64_t Paging::CreateUserAddressSpace() {
-        NotImplemented("aarch64: Paging::CreateUserAddressSpace not implemented");
+        void* page = Memory::g_pfa->AllocateZeroed();
+        if (page == nullptr) return 0;
+        return Memory::SubHHDM(page);
     }
 
-    bool Paging::MapUserIn(std::uint64_t, std::uint64_t, std::uint64_t) {
+    bool Paging::MapUserIn(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
-        NotImplemented("aarch64: Paging::MapUserIn not implemented");
+        return MapUserInPermissions(root, physicalAddress, virtualAddress, true, true);
     }
 
-    bool Paging::MapUserInPermissions(std::uint64_t, std::uint64_t, std::uint64_t, bool, bool) {
+    static bool MapUserLeaf(uint64_t root, uint64_t phys, uint64_t va, Descriptor desc, const char* what) {
-        NotImplemented("aarch64: Paging::MapUserInPermissions not implemented");
+        pagingLock.Acquire();
+        if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
+            pagingLock.Release();
+            Panic(what, nullptr);
+        }
+        volatile Descriptor* entry = WalkCreate(root, va);
+        if (entry == nullptr) {
+            pagingLock.Release();
+            return false;
+        }
+        SetLeaf(entry, desc, va);
+        pagingLock.Release();
+        return true;
     }
 
-    bool Paging::ProtectUserIn(std::uint64_t, std::uint64_t, bool, bool) {
+    bool Paging::MapUserInPermissions(std::uint64_t root, std::uint64_t physicalAddress,
-        NotImplemented("aarch64: Paging::ProtectUserIn not implemented");
+                                      std::uint64_t virtualAddress, bool writable, bool executable) {
+        return MapUserLeaf(root, physicalAddress, virtualAddress,
+                           UserLeaf(physicalAddress, AttrNormal, writable, executable),
+                           "Non-aligned address in Paging::MapUserIn!");
     }
 
-    bool Paging::MapUserInWC(std::uint64_t, std::uint64_t, std::uint64_t) {
+    bool Paging::MapUserInWC(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
-        NotImplemented("aarch64: Paging::MapUserInWC not implemented");
+        return MapUserLeaf(root, physicalAddress, virtualAddress,
+                           UserLeaf(physicalAddress, AttrNormalNc, true, false),
+                           "Non-aligned address in Paging::MapUserInWC!");
     }
 
-    void Paging::UnmapUserIn(std::uint64_t, std::uint64_t) {
+    // Permission changes need no break-before-make, only the TLB flush.
-        NotImplemented("aarch64: Paging::UnmapUserIn not implemented");
+    bool Paging::ProtectUserIn(std::uint64_t root, std::uint64_t virtualAddress,
+                               bool writable, bool executable) {
+        if ((virtualAddress & 0xFFFULL) != 0) return false;
+        pagingLock.Acquire();
+        volatile Descriptor* entry = WalkExisting(root, virtualAddress);
+        if (entry == nullptr || !IsUserLeaf(*entry)) {
+            pagingLock.Release();
+            return false;
         }
 
-    void Paging::FreeUserHalf(std::uint64_t) {
+        Descriptor desc = *entry;
-        NotImplemented("aarch64: Paging::FreeUserHalf not implemented");
+        if (writable) desc &= ~DescApReadOnly;
+        else desc |= DescApReadOnly;
+        if (executable) desc &= ~DescUxn;
+        else desc |= DescUxn;
+        *entry = desc;
+        Arch::FlushTlbPage(virtualAddress);
+        pagingLock.Release();
+        return true;
     }
 
-    bool Paging::IsUserRangeAccessible(std::uint64_t, std::uint64_t, std::uint64_t, bool) {
+    void Paging::UnmapUserIn(std::uint64_t root, std::uint64_t virtualAddress) {
-        NotImplemented("aarch64: Paging::IsUserRangeAccessible not implemented");
+        pagingLock.Acquire();
+        volatile Descriptor* entry = WalkExisting(root, virtualAddress);
+        if (entry != nullptr && (*entry & DescValid)) {
+            *entry = 0;
+            Arch::FlushTlbPage(virtualAddress);
+        }
+        pagingLock.Release();
     }
 
-    void Paging::MapEfiRuntime(const montauk::boot::EfiInfo&) {
+    void Paging::FreeUserHalf(std::uint64_t root) {
-        NotImplemented("aarch64: Paging::MapEfiRuntime not implemented");
+        // User space is the lower 128 TiB: level-0 entries 0-255.
+        for (int i0 = 0; i0 < 256; i0++) {
+            Descriptor d0 = TableAt(root)[i0];
+            if ((d0 & DescValid) == 0) continue;
+            uint64_t l1 = d0 & DescAddrMask;
+
+            for (int i1 = 0; i1 < 512; i1++) {
+                Descriptor d1 = TableAt(l1)[i1];
+                if ((d1 & DescValid) == 0) continue;
+                uint64_t l2 = d1 & DescAddrMask;
+
+                for (int i2 = 0; i2 < 512; i2++) {
+                    Descriptor d2 = TableAt(l2)[i2];
+                    if ((d2 & DescValid) == 0) continue;
+                    uint64_t l3 = d2 & DescAddrMask;
+
+                    for (int i3 = 0; i3 < 512; i3++) {
+                        Descriptor leaf = TableAt(l3)[i3];
+                        if ((leaf & DescValid) == 0) continue;
+
+                        // Write-combining/device pages are not the process's.
+                        if (((leaf & DescAttrMask) >> DescAttrShift) != AttrNormal) continue;
+
+                        uint64_t pagePhys = leaf & DescAddrMask;
+                        if (pagePhys != 0) {
+                            Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
+                        }
+                    }
+
+                    Memory::g_pfa->Free((void*)Memory::HHDM(l3));
+                }
+
+                Memory::g_pfa->Free((void*)Memory::HHDM(l2));
+            }
+
+            Memory::g_pfa->Free((void*)Memory::HHDM(l1));
+        }
+    }
+
+    bool Paging::IsUserRangeAccessible(std::uint64_t root, std::uint64_t virtualAddress,
+                                       std::uint64_t size, bool requireWrite) {
+        if (size == 0) return true;
+        if (virtualAddress >= UserSpaceEnd) return false;
+        if (virtualAddress + size < virtualAddress) return false;
+        if (virtualAddress + size > UserSpaceEnd) return false;
+
+        uint64_t page = virtualAddress & ~0xFFFULL;
+        uint64_t lastPage = (virtualAddress + size - 1) & ~0xFFFULL;
+        for (;;) {
+            volatile Descriptor* entry = WalkExisting(root, page);
+            if (entry == nullptr) return false;
+            Descriptor desc = *entry;
+            if (!IsUserLeaf(desc)) return false;
+            if (requireWrite && (desc & DescApReadOnly)) return false;
+            if (page == lastPage) break;
+            page += 0x1000;
+        }
+        return true;
     }
 
 }

kernel/src/Arch/aarch64/Platform.cpp

@@ -12,12 +12,32 @@
 
 using namespace Kt;
 
+extern "C" char ArchExceptionVectors[];  // Vectors.S
+
 namespace Arch {
 
+    // SCTLR_EL1.WXN: writable memory is execute-never
+    static constexpr uint64_t SctlrWxn = 1ULL << 19;
+
     void EarlyInitialize() {
         // No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
         asm volatile("msr tpidr_el1, xzr");
 
+        // Run on SP_EL1 so exceptions taken from the kernel arrive on the
+        // stack it is already using (Vectors.S has no SP_EL0 handlers).
+        asm volatile("mov x9, sp\n\tmsr spsel, #1\n\tmov sp, x9" ::: "x9", "memory");
+
+        // The vectors need no allocation, so take over from the
+        // bootloader's right away.
+        asm volatile("msr vbar_el1, %0\n\tisb" :: "r"(ArchExceptionVectors) : "memory");
+
+        // Kernel mappings are RWX (Paging.cpp), so writable must not
+        // imply execute-never.
+        uint64_t sctlr;
+        asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr));
+        sctlr &= ~SctlrWxn;
+        asm volatile("msr sctlr_el1, %0\n\tisb" :: "r"(sctlr));
+
         // CPACR_EL1.FPEN = 0b11: FP/SIMD usable at EL0 and EL1 (the kernel
         // itself is built -mgeneral-regs-only; Context.S saves user state).
         uint64_t cpacr;
@@ -26,10 +46,10 @@ namespace Arch {
         asm volatile("msr cpacr_el1, %0\n\tisb" :: "r"(cpacr));
     }
 
-    // TODO(aarch64): install the VBAR_EL1 vector table (EL1/EL0 synchronous
+    // Installed by EarlyInitialize.
-    // exceptions -> panic / SyscallDispatch, IRQ -> GIC -> ArchIrqDispatch).
     void InitializeTraps() {
-        KernelLogStream(WARNING, "Arch") << "aarch64: exception vectors not implemented";
+        KernelLogStream(OK, "Arch") << "Installed exception vectors at 0x" << base::hex
+            << (uint64_t)ArchExceptionVectors;
     }
 
     // MAIR_EL1 attribute indices are chosen together with the page-table

kernel/src/Arch/aarch64/SyscallFrame.hpp

@@ -9,13 +9,17 @@
 
 namespace Arch {
 
-    // Saved by the EL0 synchronous-exception vector for `svc #0`:
+    // The exception frame Vectors.S saves for `svc #0` (the same layout as
-    // x0-x5 carry the arguments and x8 the system-call number.
+    // Arch::PanicFrame): x0-x5 carry the arguments and x8 the system-call
+    // number; the result is written back to x0.
     struct SyscallFrame {
         uint64_t arg1, arg2, arg3, arg4, arg5, arg6;  // x0-x5
+        uint64_t x6, x7;
         uint64_t syscall_nr;                           // x8
-        uint64_t elr, spsr, sp_el0;                    // user PC, PSTATE, SP
+        uint64_t x9_x30[22];
-        uint64_t x19_x30[12];                          // callee-saved + LR
+        uint64_t sp_el0, elr, spsr, esr, far;
     };
 
+    static_assert(sizeof(SyscallFrame) == 288, "must match Vectors.S");
+
 }

kernel/src/Arch/aarch64/Vectors.S

@@ -0,0 +1,201 @@
+/*
+    Vectors.S
+    aarch64 exception vector table (VBAR_EL1)
+    Copyright (c) 2026 Daniel Hammer
+*/
+
+/*
+ * Every exception saves the same 288-byte frame on SP_EL1, laid out as
+ * Arch::PanicFrame (InterruptFrame.hpp) and Arch::SyscallFrame:
+ *
+ *     0   x0 .. x30       (x30 at 240)
+ *   248   interrupted SP  (SP_EL0 from EL0, the pre-exception SP_EL1 from EL1)
+ *   256   ELR_EL1
+ *   264   SPSR_EL1
+ *   272   ESR_EL1
+ *   280   FAR_EL1
+ *
+ * The kernel runs on SP_EL1 (SPSel = 1, see Platform.cpp). From EL0,
+ * SP_EL1 is the top of the thread's kernel stack: ArchEnterUserMode sets
+ * it before the first ERET and every return to EL0 restores it.
+ */
+
+#define FRAME_SIZE  288
+#define FRAME_SP    248
+#define FRAME_ELR   256
+#define FRAME_SPSR  264
+#define FRAME_ESR   272
+#define FRAME_FAR   280
+
+    .text
+
+.macro SAVE_FRAME el
+    sub     sp, sp, #FRAME_SIZE
+    stp     x0, x1, [sp, #0]
+    stp     x2, x3, [sp, #16]
+    stp     x4, x5, [sp, #32]
+    stp     x6, x7, [sp, #48]
+    stp     x8, x9, [sp, #64]
+    stp     x10, x11, [sp, #80]
+    stp     x12, x13, [sp, #96]
+    stp     x14, x15, [sp, #112]
+    stp     x16, x17, [sp, #128]
+    stp     x18, x19, [sp, #144]
+    stp     x20, x21, [sp, #160]
+    stp     x22, x23, [sp, #176]
+    stp     x24, x25, [sp, #192]
+    stp     x26, x27, [sp, #208]
+    stp     x28, x29, [sp, #224]
+    .if \el == 0
+    mrs     x21, sp_el0
+    .else
+    add     x21, sp, #FRAME_SIZE
+    .endif
+    stp     x30, x21, [sp, #240]
+    mrs     x22, elr_el1
+    mrs     x23, spsr_el1
+    stp     x22, x23, [sp, #FRAME_ELR]
+    mrs     x24, esr_el1
+    mrs     x25, far_el1
+    stp     x24, x25, [sp, #FRAME_ESR]
+.endm
+
+/* Interrupts are masked again before ELR/SPSR are reloaded: an IRQ taken
+   in between would overwrite them. */
+.macro RESTORE_FRAME el
+    msr     daifset, #2
+    ldp     x22, x23, [sp, #FRAME_ELR]
+    msr     elr_el1, x22
+    msr     spsr_el1, x23
+    .if \el == 0
+    ldr     x21, [sp, #FRAME_SP]
+    msr     sp_el0, x21
+    .endif
+    ldp     x0, x1, [sp, #0]
+    ldp     x2, x3, [sp, #16]
+    ldp     x4, x5, [sp, #32]
+    ldp     x6, x7, [sp, #48]
+    ldp     x8, x9, [sp, #64]
+    ldp     x10, x11, [sp, #80]
+    ldp     x12, x13, [sp, #96]
+    ldp     x14, x15, [sp, #112]
+    ldp     x16, x17, [sp, #128]
+    ldp     x18, x19, [sp, #144]
+    ldp     x20, x21, [sp, #160]
+    ldp     x22, x23, [sp, #176]
+    ldp     x24, x25, [sp, #192]
+    ldp     x26, x27, [sp, #208]
+    ldp     x28, x29, [sp, #224]
+    ldr     x30, [sp, #240]
+    add     sp, sp, #FRAME_SIZE
+    eret
+.endm
+
+/* Each vector slot is 128 bytes; the handlers live outside the table. */
+.macro VECTOR target
+    .balign 0x80
+    b       \target
+.endm
+
+    .balign 0x800
+    .global ArchExceptionVectors
+ArchExceptionVectors:
+    /* Current EL with SP_EL0: the kernel never runs on SP_EL0. */
+    VECTOR  el1t_unexpected
+    VECTOR  el1t_unexpected
+    VECTOR  el1t_unexpected
+    VECTOR  el1t_unexpected
+    /* Current EL with SP_ELx */
+    VECTOR  el1h_sync
+    VECTOR  el1h_irq
+    VECTOR  el1h_fiq
+    VECTOR  el1h_serror
+    /* Lower EL, AArch64 */
+    VECTOR  el0_sync
+    VECTOR  el0_irq
+    VECTOR  el0_fiq
+    VECTOR  el0_serror
+    /* Lower EL, AArch32: not supported */
+    VECTOR  el0_aarch32
+    VECTOR  el0_aarch32
+    VECTOR  el0_aarch32
+    VECTOR  el0_aarch32
+
+/* ---- Exceptions from the kernel ---- */
+
+el1t_unexpected:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #0
+    bl      ArchUnexpectedException
+
+el1h_sync:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #0
+    bl      ArchSyncException
+    RESTORE_FRAME 1
+
+el1h_irq:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #0
+    bl      ArchIrqException
+    RESTORE_FRAME 1
+
+el1h_fiq:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #1
+    bl      ArchUnexpectedException
+
+el1h_serror:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #2
+    bl      ArchUnexpectedException
+
+/* ---- Exceptions from user mode ---- */
+
+el0_sync:
+    SAVE_FRAME 0
+    lsr     x9, x24, #26            /* ESR_EL1.EC */
+    cmp     x9, #0x15               /* SVC from AArch64 */
+    b.ne    1f
+
+    msr     daifclr, #2
+    mov     x0, sp
+    bl      SyscallDispatch
+    str     x0, [sp, #0]
+    bl      SchedExitIfKilled
+    RESTORE_FRAME 0
+
+1:  mov     x0, sp
+    mov     x1, #1
+    bl      ArchSyncException
+    RESTORE_FRAME 0
+
+el0_irq:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #1
+    bl      ArchIrqException
+    RESTORE_FRAME 0
+
+el0_fiq:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #1
+    bl      ArchUnexpectedException
+
+el0_serror:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #2
+    bl      ArchUnexpectedException
+
+el0_aarch32:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #3
+    bl      ArchUnexpectedException

kernel/src/Boot/Main.cpp

@@ -27,6 +27,7 @@
 #include <Pci/Pci.hpp>
 #include <Timekeeping/Timer.hpp>
 #include <Drivers/Init.hpp>
+#include <Drivers/Serial/Uart.hpp>
 #include <Graphics/Framebuffer.hpp>
 #include <Arch/Cpu.hpp>
 #include <Arch/Platform.hpp>
@@ -111,6 +112,8 @@ extern "C" void kmain() {
 
     Hal::ACPI g_acpi((Hal::ACPI::XSDP*)Memory::HHDM(boot.rsdpPhysical));
 
+    Drivers::Serial::InitializeConsole(g_acpi.GetXSDT());
+
     if (g_acpi.GetXSDT() != nullptr) {
         Hal::AcpiShutdown::Initialize(g_acpi.GetXSDT());
 

kernel/src/Drivers/Serial/Uart.cpp

@@ -0,0 +1,181 @@
+/*
+    * Uart.cpp
+    * Polled serial console from the ACPI SPCR table
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Uart.hpp"
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Io/IoPort.hpp>
+#include <Arch/Cpu.hpp>
+
+using namespace Kt;
+
+namespace Drivers::Serial {
+
+    struct GenericAddress {
+        uint8_t  AddressSpaceId;  // 0 = system memory, 1 = system I/O
+        uint8_t  RegisterBitWidth;
+        uint8_t  RegisterBitOffset;
+        uint8_t  AccessSize;      // 1 = byte, 2 = word, 3 = dword, 4 = qword
+        uint64_t Address;
+    } __attribute__((packed));
+
+    struct Spcr {
+        Hal::ACPI::CommonSDTHeader Header;
+        uint8_t        InterfaceType;
+        uint8_t        Reserved[3];
+        GenericAddress BaseAddress;
+        uint8_t        InterruptType;
+        uint8_t        Irq;
+        uint32_t       GlobalSystemInterrupt;
+        uint8_t        BaudRate;
+        uint8_t        Parity;
+        uint8_t        StopBits;
+        uint8_t        FlowControl;
+        uint8_t        TerminalType;
+    } __attribute__((packed));
+
+    // SPCR / DBG2 serial interface subtypes
+    static constexpr uint8_t Type16550        = 0x00;
+    static constexpr uint8_t Type16450        = 0x01;
+    static constexpr uint8_t TypePl011        = 0x03;
+    static constexpr uint8_t TypeSbsa32       = 0x0D;
+    static constexpr uint8_t TypeSbsaGeneric  = 0x0E;
+    static constexpr uint8_t TypeBcm2835      = 0x10;
+    static constexpr uint8_t Type16550Generic = 0x12;
+
+    // PL011: data register, flag register (TX FIFO full)
+    static constexpr uint64_t Pl011Dr = 0x00;
+    static constexpr uint64_t Pl011Fr = 0x18;
+    static constexpr uint32_t Pl011FrTxff = 1u << 5;
+
+    // 16550: transmit holding register, line status register (THR empty)
+    static constexpr uint64_t Uart16550Thr = 0;
+    static constexpr uint64_t Uart16550Lsr = 5;
+    static constexpr uint8_t Uart16550LsrThre = 1u << 5;
+
+    // Bounded so a wedged or absent UART cannot hang the console.
+    static constexpr int TxSpinLimit = 100000;
+
+    static volatile uint8_t* g_mmio = nullptr;
+    static uint16_t g_port = 0;
+    static uint32_t g_stride = 1;
+
+    static inline uint32_t Read32(uint64_t offset) {
+        return *(volatile uint32_t*)(g_mmio + offset);
+    }
+
+    static inline void Write32(uint64_t offset, uint32_t value) {
+        *(volatile uint32_t*)(g_mmio + offset) = value;
+    }
+
+    static void Pl011Putchar(char c) {
+        for (int i = 0; i < TxSpinLimit && (Read32(Pl011Fr) & Pl011FrTxff); i++) {
+            Arch::Pause();
+        }
+        Write32(Pl011Dr, (uint8_t)c);
+    }
+
+    static uint8_t Mmio16550Read(uint64_t reg) {
+        if (g_stride == 4) return (uint8_t)Read32(reg * 4);
+        return g_mmio[reg];
+    }
+
+    static void Mmio16550Write(uint64_t reg, uint8_t value) {
+        if (g_stride == 4) Write32(reg * 4, value);
+        else g_mmio[reg] = value;
+    }
+
+    static void Mmio16550Putchar(char c) {
+        for (int i = 0; i < TxSpinLimit && !(Mmio16550Read(Uart16550Lsr) & Uart16550LsrThre); i++) {
+            Arch::Pause();
+        }
+        Mmio16550Write(Uart16550Thr, (uint8_t)c);
+    }
+
+    static void Port16550Putchar(char c) {
+        for (int i = 0; i < TxSpinLimit && !(Io::In8(g_port + Uart16550Lsr) & Uart16550LsrThre); i++) {
+            Arch::Pause();
+        }
+        Io::Out8((uint8_t)c, g_port + Uart16550Thr);
+    }
+
+    static Spcr* FindSpcr(Hal::ACPI::CommonSDTHeader* xsdt) {
+        uint32_t entryCount = (xsdt->Length - sizeof(Hal::ACPI::CommonSDTHeader)) / 8;
+        uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(Hal::ACPI::CommonSDTHeader));
+
+        for (uint32_t i = 0; i < entryCount; i++) {
+            auto* header = (Hal::ACPI::CommonSDTHeader*)Memory::HHDM(entries[i]);
+            const char* sig = header->Signature;
+            if (sig[0] == 'S' && sig[1] == 'P' && sig[2] == 'C' && sig[3] == 'R') {
+                return (Spcr*)header;
+            }
+        }
+        return nullptr;
+    }
+
+    static volatile uint8_t* MapRegisters(uint64_t phys) {
+        uint64_t page = phys & ~0xFFFULL;
+        Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
+        return (volatile uint8_t*)Memory::HHDM(phys);
+    }
+
+    void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt) {
+        if (xsdt == nullptr) return;
+
+        Spcr* spcr = FindSpcr(xsdt);
+        if (spcr == nullptr || spcr->Header.Length < sizeof(Spcr)) {
+            return;
+        }
+
+        const GenericAddress& reg = spcr->BaseAddress;
+        if (reg.Address == 0) return;
+
+        SerialSink sink = nullptr;
+        const char* name = "";
+        bool isIo = reg.AddressSpaceId == 1;
+
+        switch (spcr->InterfaceType) {
+            case TypePl011:
+            case TypeSbsa32:
+            case TypeSbsaGeneric:
+                if (isIo) return;
+                g_mmio = MapRegisters(reg.Address);
+                sink = Pl011Putchar;
+                name = "PL011";
+                break;
+
+            case Type16550:
+            case Type16450:
+            case Type16550Generic:
+            case TypeBcm2835:
+                if (isIo) {
+                    if (!Io::HasPortIo) return;
+                    g_port = (uint16_t)reg.Address;
+                    sink = Port16550Putchar;
+                } else {
+                    // The BCM2835 mini UART is a 16550 with 32-bit registers.
+                    g_stride = (reg.RegisterBitWidth == 32 || reg.AccessSize == 3
+                                || spcr->InterfaceType == TypeBcm2835) ? 4 : 1;
+                    g_mmio = MapRegisters(reg.Address);
+                    sink = Mmio16550Putchar;
+                }
+                name = spcr->InterfaceType == TypeBcm2835 ? "BCM2835 mini UART" : "16550";
+                break;
+
+            default:
+                KernelLogStream(WARNING, "Serial") << "SPCR interface type 0x" << base::hex
+                    << (uint64_t)spcr->InterfaceType << " not supported";
+                return;
+        }
+
+        AttachSerialConsole(sink);
+        KernelLogStream(OK, "Serial") << "Console on " << name << " at " << (isIo ? "port " : "")
+            << "0x" << base::hex << reg.Address;
+    }
+
+}

kernel/src/Drivers/Serial/Uart.hpp

@@ -0,0 +1,18 @@
+/*
+    * Uart.hpp
+    * Polled serial console from the ACPI SPCR table
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <ACPI/ACPI.hpp>
+
+namespace Drivers::Serial {
+
+    // Find the firmware's serial console (SPCR: PL011, SBSA generic UART,
+    // 16550 or the BCM2835 mini UART) and mirror the kernel console to it.
+    // Uses the baud rate the firmware configured. Needs the kernel page
+    // tables (the registers are mapped uncached through the HHDM).
+    void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt);
+
+}

kernel/src/Pci/Pci.cpp

@@ -187,7 +187,20 @@ namespace Pci {
         Io::Out32(value, ConfigDataPort);
     }
 
+    // Drivers reach config space through the Legacy* calls below. Without
+    // an I/O address space (ARM) there is no CF8/CFC mechanism, so they use
+    // ECAM there, at the access's own width (16/8-bit writes must not
+    // read-modify-write a neighbouring RW1C register).
+    static bool EcamCovers(uint8_t bus);
+
+    static bool UseEcamForLegacy(uint8_t bus) {
+        return !Io::HasPortIo && EcamCovers(bus);
+    }
+
     uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
+        if (UseEcamForLegacy(bus)) {
+            return EcamRead32(g_ecamBase, bus, device, function, offset & 0xFC);
+        }
         g_legacyConfigLock.Acquire();
         uint32_t value = LegacyRead32Unlocked(bus, device, function, offset);
         g_legacyConfigLock.Release();
@@ -205,12 +218,20 @@ namespace Pci {
     }
 
     void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) {
+        if (UseEcamForLegacy(bus)) {
+            EcamWrite32(g_ecamBase, bus, device, function, offset & 0xFC, value);
+            return;
+        }
         g_legacyConfigLock.Acquire();
         LegacyWrite32Unlocked(bus, device, function, offset, value);
         g_legacyConfigLock.Release();
     }
 
     void LegacyWrite16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint16_t value) {
+        if (UseEcamForLegacy(bus)) {
+            EcamWrite16(g_ecamBase, bus, device, function, offset & 0xFE, value);
+            return;
+        }
         g_legacyConfigLock.Acquire();
         uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
         int shift = (offset & 2) * 8;
@@ -221,6 +242,10 @@ namespace Pci {
     }
 
     void LegacyWrite8(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint8_t value) {
+        if (UseEcamForLegacy(bus)) {
+            EcamWrite8(g_ecamBase, bus, device, function, offset, value);
+            return;
+        }
         g_legacyConfigLock.Acquire();
         uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC);
         int shift = (offset & 3) * 8;

kernel/src/Terminal/Terminal.cpp

@@ -37,6 +37,14 @@ namespace Kt {
     // ANSI escape sequence filter state
     static bool g_ansiEscape = false;
 
+    static SerialSink g_serialSink = nullptr;
+
+    static void SerialPutchar(char c) {
+        if (g_serialSink == nullptr) return;
+        if (c == '\n') g_serialSink('\r');
+        g_serialSink(c);
+    }
+
     static void RingBufferAppend(char c) {
         // Strip ANSI escape sequences
         if (c == '\033') {
@@ -209,18 +217,20 @@ namespace Kt {
             // framebuffer terminal below; storing it in the ring makes file
             // consumers treat one logical newline as two line breaks.
             RingBufferAppend(c);
+            SerialPutchar(c);
 
             if (g_suppressKernelLog) {
                 return;
             }
-        }
+        } else {
-
             // Once a graphical app takes over, suppress ALL flanterm writes
             // (SYS_PRINT from user processes, etc.) to avoid painting text
             // over the GUI framebuffer.
-        if (g_suppressKernelLog && g_kernelLogDepth == 0) {
+            if (g_suppressKernelLog) {
                 return;
             }
+            SerialPutchar(c);
+        }
 
         if (c == '\n') {
             flanterm_write(ctx, "\r\n", 2);
@@ -235,6 +245,18 @@ namespace Kt {
         }
     }
 
+    void AttachSerialConsole(SerialSink sink) {
+        g_termLock.Acquire();
+        uint64_t start = g_klogCount < KLOG_BUF_SIZE ? 0 : g_klogHead;
+        for (uint64_t i = 0; i < g_klogCount; i++) {
+            char c = g_klogBuf[(start + i) % KLOG_BUF_SIZE];
+            if (c == '\n') sink('\r');
+            sink(c);
+        }
+        g_serialSink = sink;
+        g_termLock.Release();
+    }
+
     void SuppressKernelLog() {
         g_suppressKernelLog = true;
     }

kernel/src/Terminal/Terminal.hpp

@@ -43,6 +43,12 @@ namespace Kt
     void Putchar(char c);
     void Print(const char *text);
 
+    // Mirror console output to a polled serial port (Drivers/Serial). The
+    // sink receives the whole kernel log, also once the desktop suppresses
+    // it on screen; what was logged before attaching is replayed first.
+    using SerialSink = void (*)(char c);
+    void AttachSerialConsole(SerialSink sink);
+
     // Terminal mutex - protects flanterm from concurrent CPU access.
     // Uses Mutex (not Spinlock) to keep interrupts enabled while held,
     // preventing dropped PS/2 mouse/keyboard bytes during log output.

scripts/check-arch-neutral.sh

@@ -93,8 +93,10 @@ if [ -n "$missing" ]; then
 fi
 echo "check-arch-neutral: $ARCH objects compile and every symbol resolves"
 
-if command -v "${KERNEL_LD:-ld.lld}" >/dev/null 2>&1; then
+# Same lookup as kernel/GNUmakefile: a system ld.lld, else rustup's.
-    make ARCH="$ARCH" >/dev/null
+lld="${KERNEL_LD:-$(command -v ld.lld || ls "$HOME"/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld 2>/dev/null | head -n 1)}"
+if [ -n "$lld" ] && command -v "$lld" >/dev/null 2>&1; then
+    make ARCH="$ARCH" KERNEL_LD="$lld" >/dev/null
     echo "check-arch-neutral: linked kernel/bin-$ARCH/kernel"
 fi