diff --git a/GNUmakefile b/GNUmakefile index 3a1416b..f2c0906 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -10,6 +10,22 @@ QEMUFLAGS := -smp 4 -m 2G -d int -D qemu.log -no-reboot override IMAGE_NAME := montauk-$(ARCH) +# Userspace exists only for x86_64 so far: other architectures get a +# ramdisk holding just the boot files. +ifeq ($(ARCH),x86_64) + KERNEL_ONLY ?= 0 +else + KERNEL_ONLY ?= 1 +endif + +override EFI_LOADER_x86_64 := BOOTX64.EFI +override EFI_LOADER_aarch64 := BOOTAA64.EFI +override EFI_LOADER_riscv64 := BOOTRISCV64.EFI +override EFI_LOADER_loongarch64 := BOOTLOONGARCH64.EFI + +# AArch64 UEFI firmware for QEMU (Debian/Ubuntu: qemu-efi-aarch64). +AARCH64_EFI_FIRMWARE ?= /usr/share/qemu-efi-aarch64/QEMU_EFI.fd + # Toolchain for building the 'limine' executable for the host. HOST_CC := cc HOST_CFLAGS := -g -O2 -pipe @@ -51,29 +67,34 @@ run-hdd-x86_64: $(IMAGE_NAME).hdd $(QEMUFLAGS) .PHONY: run-aarch64 +# The machine mirrors the Raspberry Pi 4/400: Cortex-A72 cores and a GICv2. +# The kernel console is mirrored to the PL011 (-serial stdio). run-aarch64: $(IMAGE_NAME).iso qemu-system-$(ARCH) \ - -M virt \ + -M virt,gic-version=2 \ -cpu cortex-a72 \ -device ramfb \ -device qemu-xhci,id=xhci \ -device usb-kbd,bus=xhci.0 \ -device usb-mouse,bus=xhci.0 \ - -bios /usr/share/ovmf/OVMF.fd \ + -bios $(AARCH64_EFI_FIRMWARE) \ -cdrom $(IMAGE_NAME).iso \ + -serial stdio \ $(QEMUFLAGS) .PHONY: run-hdd-aarch64 run-hdd-aarch64: $(IMAGE_NAME).hdd qemu-system-$(ARCH) \ - -M virt \ + -M virt,gic-version=2 \ -cpu cortex-a72 \ -device ramfb \ -device qemu-xhci,id=xhci \ -device usb-kbd,bus=xhci.0 \ -device usb-mouse,bus=xhci.0 \ - -bios /usr/share/ovmf/OVMF.fd \ - -hda $(IMAGE_NAME).hdd \ + -bios $(AARCH64_EFI_FIRMWARE) \ + -drive if=none,id=hd0,format=raw,file=$(IMAGE_NAME).hdd \ + -device virtio-blk-pci,drive=hd0 \ + -serial stdio \ $(QEMUFLAGS) .PHONY: run-riscv64 @@ -196,13 +217,23 @@ ports: programs fi .PHONY: ramdisk +ifeq ($(KERNEL_ONLY),1) +ramdisk: limine/limine kernel + rm -rf .kernel-only-root + mkdir -p .kernel-only-root/boot/limine + cp -v limine/$(EFI_LOADER_$(ARCH)) limine.conf .kernel-only-root/boot/limine/ + cp -v kernel/bin-$(ARCH)/kernel .kernel-only-root/boot/ + ./scripts/mkramdisk.sh .kernel-only-root ramdisk.tar + rm -rf .kernel-only-root +else ramdisk: limine/limine kernel programs mkdir -p programs/bin/boot/limine - cp -v limine/BOOTX64.EFI programs/bin/boot/limine/ + cp -v limine/$(EFI_LOADER_$(ARCH)) programs/bin/boot/limine/ cp -v limine.conf programs/bin/boot/limine/ cp -v kernel/bin-$(ARCH)/kernel programs/bin/boot/ rm -f programs/bin/boot/ramdisk.tar ./scripts/mkramdisk.sh programs/bin ramdisk.tar +endif $(IMAGE_NAME).iso: limine/limine kernel ramdisk rm -rf iso_root diff --git a/docs/porting.md b/docs/porting.md index e4053e8..b24eea2 100644 --- a/docs/porting.md +++ b/docs/porting.md @@ -36,7 +36,7 @@ kernel/src/Arch/ Platform.hpp boot-sequence hooks, platform device list, reboot CpuPower.hpp frequency scaling / thermal governor (may be no-ops) x86_64/ the complete port - aarch64/ skeleton port + aarch64/ the port in progress (see below) ``` Other interfaces a port implements: @@ -85,25 +85,48 @@ are `Cpu.hpp`, `IoPort.hpp`, `PerCpu.hpp`, `Context.hpp`, `Abi.hpp` and - Interrupts are numbered IRQ slots (`Arch::IRQ_COUNT`); slot numbers for MSI-capable drivers are fixed in the drivers. -## The aarch64 skeleton +## The aarch64 port Real: CPU primitives (DAIF, WFI, `CNTVCT_EL0`, TLBI, TTBR0), per-CPU data via `TPIDR_EL1`, the context switch and EL0 entry (`Context.S`), TLS through `TPIDR_EL0`, the ELF/TLS/exit-stub ABI (syscall number in x8), the IRQ slot -dispatch table (`ArchIrqDispatch`), panic-frame printing. +dispatch table (`ArchIrqDispatch`), panic-frame printing, and: + +- Exception vectors (`Vectors.S`, installed in `EarlyInitialize`). Every + exception saves one 288-byte frame that is both `Arch::PanicFrame` and + `Arch::SyscallFrame`. EL0 `svc` goes to `SyscallDispatch`; other + synchronous exceptions go to `ArchSyncException` (`Exceptions.cpp`), which + grows user stacks and lazy anonymous memory, kills faulting processes with + a crash report, and panics on kernel faults. +- Translation tables (`Paging.cpp`): 4 KiB granule, 48-bit VAs. The kernel + half lives in its own TTBR1 table; `KernelRoot()` is the TTBR0 table of + kernel contexts (EFI identity mappings only, so null pointers fault). The + HHDM maps only the memory map's RAM and firmware regions, never the holes + between them (device space on ARM). MAIR: 0 Normal WB, 1 Normal NC + (`MapWC`), 2 Device-nGnRE (`MapMMIO`). Not yet written (each is marked `TODO(aarch64)`): -- `Paging.cpp`: translation tables and MAIR (every method panics). -- Exception vectors (`VBAR_EL1`): synchronous faults, `svc`, IRQs. - GIC driver: SPI routing, SGIs for IPIs, MSIs via ITS/GICv2m - (`ComposeMsi` returns false, so drivers fall back to INTx/polling). -- Generic timer tick (`Arch::Timer::InitializeBsp` returns false). + (`ComposeMsi` returns false, so drivers fall back to INTx/polling). IRQs + reach `ArchIrqException`, which only logs them. +- Generic timer tick (`Arch::Timer::InitializeBsp` returns false). Boot + currently stops in the first driver that waits on `GetMilliseconds()` + (xHCI port power-up). - Secondary CPU start-up, PSCI reset and suspend. -Building it needs `clang` and `ld.lld` (`make kernel ARCH=aarch64`; -`KERNEL_LD=` points at a different `ld.lld`, e.g. rustup's -`lib/rustlib/x86_64-unknown-linux-gnu/bin/gcc-ld/ld.lld`). +Building it needs `clang` and `ld.lld`; without a system `ld.lld` the +kernel makefile uses the one rustup ships +(`~/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld`), and `KERNEL_LD=` +overrides both. `make run ARCH=aarch64` boots a kernel-only image (userspace +is x86_64-only so far) on QEMU `virt` with a GICv2 and Cortex-A72 cores, the +Raspberry Pi 4/400's configuration; it needs `qemu-system-aarch64` and +`qemu-efi-aarch64`. + +The kernel console is mirrored to the serial port the ACPI SPCR table names +(PL011, SBSA UART, 16550 or the BCM2835 mini UART; `Drivers/Serial/Uart.cpp`), +including the log written before the port was found. On QEMU that is +`-serial stdio`. ## Adding another architecture diff --git a/kernel/GNUmakefile b/kernel/GNUmakefile index 5a9de63..e3564c5 100644 --- a/kernel/GNUmakefile +++ b/kernel/GNUmakefile @@ -291,7 +291,9 @@ cc-runtime-$(ARCH)/cc-runtime.a: GNUmakefile cc-runtime/* # so a clang-built non-x86 kernel is linked by ld.lld directly. ifneq ($(ARCH),x86_64) ifeq ($(CXX_IS_CLANG),1) - KERNEL_LD ?= ld.lld + # Without a system lld, fall back to the one rustup ships. + KERNEL_LD ?= $(firstword $(shell command -v ld.lld) \ + $(wildcard $(HOME)/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld)) override LD_EMULATION_aarch64 := aarch64elf override LD_EMULATION_riscv64 := elf64lriscv override LD_EMULATION_loongarch64 := elf64loongarch diff --git a/kernel/src/Api/BuildNo.hpp b/kernel/src/Api/BuildNo.hpp index 676bf2f..0a5fc31 100644 --- a/kernel/src/Api/BuildNo.hpp +++ b/kernel/src/Api/BuildNo.hpp @@ -12,4 +12,4 @@ #pragma once -#define MONTAUK_BUILD_NUMBER 26 +#define MONTAUK_BUILD_NUMBER 29 diff --git a/kernel/src/Arch/aarch64/Cpu.hpp b/kernel/src/Arch/aarch64/Cpu.hpp index 99fb158..da0475c 100644 --- a/kernel/src/Arch/aarch64/Cpu.hpp +++ b/kernel/src/Arch/aarch64/Cpu.hpp @@ -92,9 +92,10 @@ namespace Arch { asm volatile("dsb sy\n\tisb" ::: "memory"); } + // The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0. inline void FlushTlbPage(uint64_t virtualAddress) { asm volatile("dsb ishst\n\ttlbi vaae1is, %0\n\tdsb ish\n\tisb" - :: "r"(virtualAddress >> 12) : "memory"); + :: "r"((virtualAddress >> 12) & ((1ULL << 44) - 1)) : "memory"); } inline void FlushTlbAll() { diff --git a/kernel/src/Arch/aarch64/Exceptions.cpp b/kernel/src/Arch/aarch64/Exceptions.cpp new file mode 100644 index 0000000..6b9cfe7 --- /dev/null +++ b/kernel/src/Arch/aarch64/Exceptions.cpp @@ -0,0 +1,184 @@ +/* + * Exceptions.cpp + * aarch64 synchronous exceptions: faults, user crashes, kernel panics + * Copyright (c) 2026 Daniel Hammer +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Kt; + +namespace { + + // ESR_EL1.EC exception classes + constexpr uint64_t EcUnknown = 0x00; + constexpr uint64_t EcIllegalState = 0x0E; + constexpr uint64_t EcInstAbortLower = 0x20; + constexpr uint64_t EcInstAbortSame = 0x21; + constexpr uint64_t EcPcAlignment = 0x22; + constexpr uint64_t EcDataAbortLower = 0x24; + constexpr uint64_t EcDataAbortSame = 0x25; + constexpr uint64_t EcSpAlignment = 0x26; + constexpr uint64_t EcFpException = 0x2C; + constexpr uint64_t EcBreakpointLower = 0x30; + constexpr uint64_t EcBrk = 0x3C; + + constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL; + + uint64_t ExceptionClass(const Arch::PanicFrame* frame) { + return (frame->esr >> 26) & 0x3F; + } + + bool IsAbort(uint64_t ec) { + return ec == EcInstAbortLower || ec == EcInstAbortSame + || ec == EcDataAbortLower || ec == EcDataAbortSame; + } + + // Fault status code (ISS[5:0]) of an abort: translation faults mean + // the page is not mapped, permission faults that it is. + bool IsTranslationFault(uint64_t esr) { + uint64_t fsc = esr & 0x3F; + return fsc >= 0x04 && fsc <= 0x07; + } + + bool IsPermissionFault(uint64_t esr) { + uint64_t fsc = esr & 0x3F; + return fsc >= 0x0C && fsc <= 0x0F; + } + + const char* ExceptionName(uint64_t ec) { + switch (ec) { + case EcUnknown: return "Undefined Instruction"; + case EcIllegalState: return "Illegal Execution State"; + case EcInstAbortLower: + case EcInstAbortSame: return "Instruction Abort"; + case EcPcAlignment: return "PC Alignment Fault"; + case EcDataAbortLower: + case EcDataAbortSame: return "Data Abort"; + case EcSpAlignment: return "SP Alignment Fault"; + case EcFpException: return "Floating-Point Exception"; + case EcBrk: return "Breakpoint (BRK)"; + default: + if (ec >= EcBreakpointLower && ec <= 0x35) return "Debug Exception"; + return "Synchronous Exception"; + } + } + + TerminationCodes TerminationFromClass(uint64_t ec) { + if (ec == EcUnknown || ec == EcIllegalState) return TerminatedIllegalInstruction; + if (ec == EcBrk || (ec >= EcBreakpointLower && ec <= 0x35)) return TerminatedBreakpoint; + if (ec == EcFpException) return TerminatedArithmeticError; + return TerminatedAccessViolation; + } + + template + void CopyName(char (&dst)[N], const char* src) { + size_t i = 0; + for (; i + 1 < N && src[i]; i++) dst[i] = src[i]; + dst[i] = '\0'; + } + + // Kill the faulting process with a crash report. Never returns. + [[noreturn]] void TerminateUserProcess(Arch::PanicFrame* frame, uint64_t ec) { + // Teardown can wait on devices and other CPUs; keep the clock running. + Arch::EnableInterrupts(); + + auto* proc = Sched::GetCurrentProcessPtr(); + const char* name = ExceptionName(ec); + KernelLogStream(ERROR, "Exception") << name << " in process \"" << proc->name + << "\" (pid " << proc->pid << ", ESR 0x" << base::hex << frame->esr + << ", PC 0x" << frame->elr << ") - process terminated"; + + CrashReport::Report rep{}; + rep.valid = true; + rep.pid = proc->pid; + CopyName(rep.processName, proc->name); + rep.exceptionVector = (uint8_t)ec; + CopyName(rep.exceptionName, name); + rep.instructionPointer = frame->elr; + rep.flags = frame->spsr; + rep.stackPointer = frame->sp; + + constexpr uint64_t snapshotBytes = CrashReport::StackSnapshotWords * sizeof(uint64_t); + if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pageTableRoot, frame->sp, + snapshotBytes, false)) { + auto* userStack = (const volatile uint64_t*)frame->sp; + for (int word = 0; word < CrashReport::StackSnapshotWords; word++) + rep.stackWords[word] = userStack[word]; + rep.stackWordCount = CrashReport::StackSnapshotWords; + } + + if (IsAbort(ec)) { + rep.faultingAddress = frame->far; + rep.pfPresent = IsPermissionFault(frame->esr); + rep.pfWrite = ec == EcDataAbortLower && (frame->esr & (1ULL << 6)) != 0; + rep.pfUser = 1; + rep.pfInstructionFetch = ec == EcInstAbortLower; + } + rep.timestampTick = Timekeeping::GetTicks(); + + CrashReport::AddReport(rep); + Sched::SpawnCrashPad(proc->pid); + + // As on x86: a sibling thread's fault must end the whole process. + int code = TerminationFromClass(ec); + if (Sched::GetCurrentThreadPtr() != proc) { + Sched::KillProcess(proc->pid, code); + } else { + Sched::SetProcessExitCode(proc->pid, code); + } + + Sched::ExitProcess(); + __builtin_unreachable(); + } + + // A not-present user page: anonymous memory mapped lazily or the user + // stack growing. Also reached from the kernel touching user buffers. + bool TryResolveUserFault(const Arch::PanicFrame* frame, uint64_t ec) { + if (!IsAbort(ec) || !IsTranslationFault(frame->esr)) return false; + if (frame->far >= UserSpaceEnd) return false; + + auto* cpu = Smp::TryGetCurrentCpuData(); + if (cpu == nullptr || cpu->currentSlot < 0 || Sched::GetCurrentPid() < 0) return false; + + bool isData = ec == EcDataAbortLower || ec == EcDataAbortSame; + Memory::VMM::PageFault fault{false, isData && (frame->esr & (1ULL << 6)) != 0, !isData}; + return montauk::abi::TryHandleAnonymousPageFault(frame->far, fault) + || Sched::TryGrowUserStack(frame->far); + } + +} + +extern "C" void ArchSyncException(Arch::PanicFrame* frame, bool fromUser) { + uint64_t ec = ExceptionClass(frame); + + if (TryResolveUserFault(frame, ec)) { + return; + } + + if (fromUser && Sched::GetCurrentPid() >= 0) { + TerminateUserProcess(frame, ec); + } + + Panic(ExceptionName(ec), frame); +} + +// FIQs are never routed, SErrors are asynchronous external aborts, and the +// kernel neither runs on SP_EL0 nor supports AArch32 user code. +extern "C" void ArchUnexpectedException(Arch::PanicFrame* frame, uint64_t kind) { + static const char* const names[] = { + "Exception on SP_EL0", "Unexpected FIQ", "SError (asynchronous abort)", + "AArch32 exception", + }; + Panic(kind < 4 ? names[kind] : "Unexpected exception", frame); +} diff --git a/kernel/src/Arch/aarch64/InterruptFrame.hpp b/kernel/src/Arch/aarch64/InterruptFrame.hpp index 6a785c7..a4b1d15 100644 --- a/kernel/src/Arch/aarch64/InterruptFrame.hpp +++ b/kernel/src/Arch/aarch64/InterruptFrame.hpp @@ -9,7 +9,7 @@ namespace Arch { - // What the exception vectors save before calling into C++. + // What the exception vectors (Vectors.S) save before calling into C++. struct PanicFrame { uint64_t x[31]; // x0-x30 uint64_t sp; // interrupted SP (SP_EL0 for EL0, else SP_EL1) @@ -19,4 +19,6 @@ namespace Arch { uint64_t far; // FAR_EL1: faulting address }; + static_assert(sizeof(PanicFrame) == 288, "must match Vectors.S"); + } diff --git a/kernel/src/Arch/aarch64/Interrupts.cpp b/kernel/src/Arch/aarch64/Interrupts.cpp index 0f099e5..256fcda 100644 --- a/kernel/src/Arch/aarch64/Interrupts.cpp +++ b/kernel/src/Arch/aarch64/Interrupts.cpp @@ -45,6 +45,17 @@ namespace Arch { } +// IRQ exception vector (Vectors.S). +// TODO(aarch64): acknowledge through the GIC CPU interface, translate the +// INTID to a slot and call ArchIrqDispatch. +extern "C" void ArchIrqException(void*, bool) { + static bool reported = false; + if (!reported) { + reported = true; + Kt::KernelLogStream(Kt::WARNING, "IRQ") << "aarch64: IRQ taken with no interrupt controller driver"; + } +} + extern "C" void ArchIrqDispatch(uint64_t irq, bool fromUser) { if (irq < Arch::IRQ_COUNT && Arch::g_irqHandlers[irq] != nullptr) { Arch::g_irqHandlers[irq]((uint8_t)irq, fromUser); diff --git a/kernel/src/Arch/aarch64/Paging.cpp b/kernel/src/Arch/aarch64/Paging.cpp index 43edb46..2a6b5f3 100644 --- a/kernel/src/Arch/aarch64/Paging.cpp +++ b/kernel/src/Arch/aarch64/Paging.cpp @@ -5,29 +5,67 @@ */ #include +#include +#include #include #include #include -// TODO(aarch64): 4 KiB-granule, 48-bit VA translation tables. The kernel -// half lives in TTBR1_EL1 and is never switched; "address space roots" -// handed to the generic kernel are TTBR0_EL1 tables, so -// CreateUserAddressSpace has no kernel half to copy. MAIR_EL1 needs -// Normal-WB, Normal-NC (MapWC) and Device-nGnRE (MapMMIO) indices. +// 4 KiB granule, 48-bit virtual addresses, four levels (0-3). The kernel +// half is translated through TTBR1_EL1 (g_kernelHalfRoot), which never +// changes after Init. The "address space roots" the generic kernel switches +// between are TTBR0_EL1 tables: a process's root holds only its own lower +// half, and KernelRoot() is the TTBR0 table of kernel contexts, empty apart +// from the EFI runtime identity mappings (so null pointers fault). +// +// The HHDM covers only what the memory map lists as RAM or firmware data: +// unlike on x86, the holes between are device space on ARM machines, and a +// cacheable mapping of device registers invites speculative accesses. namespace Memory::VMM { Paging* g_paging = nullptr; + static std::uint64_t g_hhdmLimit = 0; + + static std::uint64_t g_kernelHalfRoot = 0; + + std::uint64_t Paging::HhdmLimit() { return g_hhdmLimit; } + + // Translation table descriptor bits (VMSAv8-64, stage 1) + static constexpr uint64_t DescValid = 1ULL << 0; + static constexpr uint64_t DescTable = 1ULL << 1; // table (levels 0-2) or page (level 3) + static constexpr uint64_t DescAttrShift = 2; // AttrIndx[2:0], into MAIR_EL1 + static constexpr uint64_t DescAttrMask = 7ULL << DescAttrShift; + static constexpr uint64_t DescApUser = 1ULL << 6; // AP[1]: EL0 may access + static constexpr uint64_t DescApReadOnly = 1ULL << 7; // AP[2] + static constexpr uint64_t DescShInner = 3ULL << 8; + static constexpr uint64_t DescAf = 1ULL << 10; // access flag (no AF faults) + static constexpr uint64_t DescNotGlobal = 1ULL << 11; + static constexpr uint64_t DescPxn = 1ULL << 53; + static constexpr uint64_t DescUxn = 1ULL << 54; + static constexpr uint64_t DescAddrMask = 0x0000FFFFFFFFF000ULL; + + // MAIR_EL1 attribute indices + static constexpr uint64_t AttrNormal = 0; // write-back, read/write-allocate + static constexpr uint64_t AttrNormalNc = 1; // non-cacheable (write-combining) + static constexpr uint64_t AttrDevice = 2; // Device-nGnRE + static constexpr uint64_t MairValue = 0xFFULL << (8 * AttrNormal) + | 0x44ULL << (8 * AttrNormalNc) + | 0x04ULL << (8 * AttrDevice); + + static constexpr uint64_t KernelHalfBase = 0xFFFF000000000000ULL; + static constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL; + + // EFI memory types whose runtime regions are device registers + static constexpr uint32_t EfiMemoryMappedIo = 11; + static constexpr uint32_t EfiMemoryMappedIoPortSpace = 12; + static kcp::Mutex pagingLock; void LockUserPaging() { pagingLock.Acquire(); } void UnlockUserPaging() { pagingLock.Release(); } - [[noreturn]] static void NotImplemented(const char* what) { - Panic(what, nullptr); - } - void ActivateKernelAddressSpace() { Arch::WritePageTableRoot(g_paging->KernelRoot()); } @@ -36,75 +74,431 @@ namespace Memory::VMM { Arch::FlushTlbAll(); } + using Descriptor = uint64_t; + + static inline volatile Descriptor* TableAt(uint64_t phys) { + return (volatile Descriptor*)Memory::HHDM(phys); + } + + static inline uint64_t Index(uint64_t va, int level) { + return (va >> (39 - 9 * level)) & 0x1FF; + } + + // Order descriptor writes before the table walker (and later + // instructions) can observe them. + static inline void PublishTables() { + asm volatile("dsb ishst\n\tisb" ::: "memory"); + } + + static inline uint64_t ReadTtbr1() { + uint64_t ttbr; + asm volatile("mrs %0, ttbr1_el1" : "=r"(ttbr)); + return ttbr & Arch::TtbrAddressMask; + } + + // The level-3 descriptor for `va`, creating intermediate tables. + // Returns nullptr when out of memory or when a block mapping covers + // `va` (only the bootloader's tables contain blocks). + static volatile Descriptor* WalkCreate(uint64_t root, uint64_t va) { + uint64_t table = root; + for (int level = 0; level < 3; level++) { + volatile Descriptor* entry = &TableAt(table)[Index(va, level)]; + Descriptor desc = *entry; + if ((desc & DescValid) == 0) { + void* page = Memory::g_pfa->AllocateZeroed(); + if (page == nullptr) return nullptr; + table = Memory::SubHHDM(page); + // The zeroed table must be visible before the walker can + // reach it. + asm volatile("dsb ishst" ::: "memory"); + *entry = table | DescTable | DescValid; + } else if ((desc & DescTable) == 0) { + return nullptr; + } else { + table = desc & DescAddrMask; + } + } + return &TableAt(table)[Index(va, 3)]; + } + + // The level-3 descriptor for `va`, or nullptr if a table is missing. + static volatile Descriptor* WalkExisting(uint64_t root, uint64_t va) { + uint64_t table = root; + for (int level = 0; level < 3; level++) { + Descriptor desc = TableAt(table)[Index(va, level)]; + if ((desc & (DescValid | DescTable)) != (DescValid | DescTable)) return nullptr; + table = desc & DescAddrMask; + } + return &TableAt(table)[Index(va, 3)]; + } + + // Install a leaf. Replacing a live translation (other address or + // attributes) needs break-before-make: invalidate, flush, then write. + static void SetLeaf(volatile Descriptor* entry, Descriptor desc, uint64_t va) { + Descriptor old = *entry; + if (old == desc) return; + if (old & DescValid) { + *entry = 0; + Arch::FlushTlbPage(va); + } + *entry = desc; + PublishTables(); + } + + static Descriptor KernelLeaf(uint64_t phys, uint64_t attr) { + Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner + | (attr << DescAttrShift) | DescUxn; + // Kernel memory stays executable, as on x86; device memory never is. + if (attr != AttrNormal) desc |= DescPxn; + return desc; + } + + static Descriptor UserLeaf(uint64_t phys, uint64_t attr, bool writable, bool executable) { + Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner + | DescNotGlobal | (attr << DescAttrShift) | DescApUser | DescPxn; + if (!writable) desc |= DescApReadOnly; + if (!executable) desc |= DescUxn; + return desc; + } + + static inline bool IsUserLeaf(Descriptor desc) { + return (desc & (DescValid | DescTable)) == (DescValid | DescTable) && (desc & DescApUser); + } + + // Translate through a table tree that may contain block mappings (the + // bootloader's). Returns 0 if unmapped. + static uint64_t Translate(uint64_t root, uint64_t va) { + uint64_t table = root; + for (int level = 0; level <= 3; level++) { + Descriptor desc = TableAt(table)[Index(va, level)]; + if ((desc & DescValid) == 0) return 0; + + uint64_t span = 1ULL << (39 - 9 * level); + bool leaf = level == 3 || (desc & DescTable) == 0; + if (leaf) { + if (level == 0) return 0; // no level-0 blocks with 4 KiB granules + return ((desc & DescAddrMask) & ~(span - 1)) | (va & (span - 1)); + } + table = desc & DescAddrMask; + } + return 0; + } + + void Paging::MapEfiRuntime(const montauk::boot::EfiInfo& efi) { + if (efi.memoryMap == nullptr || efi.descriptorSize == 0) return; + + struct EfiMemDesc { + uint32_t Type; + uint64_t PhysicalStart; + uint64_t VirtualStart; + uint64_t NumberOfPages; + uint64_t Attribute; + }; + + static constexpr uint64_t EFI_MEMORY_RUNTIME = 0x8000000000000000ULL; + + auto* base = (uint8_t*)efi.memoryMap; + uint64_t count = efi.memoryMapSize / efi.descriptorSize; + + for (uint64_t i = 0; i < count; i++) { + auto* desc = (EfiMemDesc*)(base + i * efi.descriptorSize); + if ((desc->Attribute & EFI_MEMORY_RUNTIME) == 0) continue; + + bool device = desc->Type == EfiMemoryMappedIo || desc->Type == EfiMemoryMappedIoPortSpace; + for (uint64_t p = 0; p < desc->NumberOfPages; p++) { + uint64_t addr = desc->PhysicalStart + p * 0x1000; + // Firmware code runs at its HHDM alias (Efi::Init calls the + // services there) and reaches its data at physical addresses. + // Runtime regions are reserved memory, outside the HHDM. + if (device) { + MapMMIO(addr, addr); + MapMMIO(addr, Memory::HHDM(addr)); + } else { + Map(addr, addr); + Map(addr, Memory::HHDM(addr)); + } + } + } + + FlushTLB(); + } + Paging::Paging() { - NotImplemented("aarch64: virtual memory not implemented"); + rootPhys = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed()); + g_kernelHalfRoot = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed()); } - void Paging::Init(std::uint64_t, std::uint64_t, const montauk::boot::MemoryMap&, - const montauk::boot::Framebuffer&) { - NotImplemented("aarch64: Paging::Init not implemented"); + // Kernel mappings of lower-half addresses (EFI identity mappings) go to + // the kernel contexts' TTBR0 table. + static uint64_t KernelRootFor(uint64_t kernelContextRoot, uint64_t va) { + return va >= KernelHalfBase ? g_kernelHalfRoot : kernelContextRoot; } - std::uint64_t Paging::HhdmLimit() { return 0; } + void Paging::Init(std::uint64_t kernelBaseVirt, std::uint64_t kernelSize, const montauk::boot::MemoryMap& memMap, + const montauk::boot::Framebuffer& framebuffer) { + if (Memory::HHDMBase < KernelHalfBase) { + Panic("aarch64: HHDM outside the 48-bit kernel half", nullptr); + } - void Paging::Map(std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::Map not implemented"); + uint64_t bootTtbr1 = ReadTtbr1(); + uint64_t bootTcr, bootMair; + asm volatile("mrs %0, tcr_el1" : "=r"(bootTcr)); + asm volatile("mrs %0, mair_el1" : "=r"(bootMair)); + Kt::KernelLogStream(Kt::DEBUG, "VMM") << "Bootloader TCR 0x" << base::hex << bootTcr + << ", MAIR 0x" << bootMair << ", TTBR1 0x" << bootTtbr1; + + // The kernel image, at the physical pages the bootloader loaded it to. + for (uint64_t va = kernelBaseVirt; va < kernelBaseVirt + kernelSize; va += 0x1000) { + uint64_t phys = Translate(bootTtbr1, va); + if (phys == 0) Panic("aarch64: kernel page missing from the bootloader's tables", nullptr); + Map(phys, va); + } + + // HHDM: every region that is real backing store (see the top of + // this file). The limit is published for the AML interpreter. + uint64_t maxPhysAddr = 0; + for (size_t i = 0; i < memMap.count; i++) { + const auto& region = memMap.regions[i]; + if (region.kind == montauk::boot::MemoryKind::Reserved + || region.kind == montauk::boot::MemoryKind::Unknown) { + continue; + } + + uint64_t start = region.base & ~0xFFFULL; + uint64_t end = (region.base + region.length + 0xFFF) & ~0xFFFULL; + bool framebufferRegion = region.kind == montauk::boot::MemoryKind::Framebuffer; + for (uint64_t phys = start; phys < end; phys += 0x1000) { + if (framebufferRegion) MapWC(phys, Memory::HHDM(phys)); + else Map(phys, Memory::HHDM(phys)); + } + if (end > maxPhysAddr) maxPhysAddr = end; + } + g_hhdmLimit = maxPhysAddr; + + // The framebuffer may not be listed in the memory map at all. + if (framebuffer.address != 0) { + uint64_t fbPhys = Memory::SubHHDM(framebuffer.address) & ~0xFFFULL; + uint64_t fbPages = (framebuffer.height * framebuffer.pitch + 0xFFF) / 0x1000; + for (uint64_t p = 0; p < fbPages; p++) { + uint64_t phys = fbPhys + p * 0x1000; + MapWC(phys, Memory::HHDM(phys)); + } + } + + // 48-bit VAs in both halves, 4 KiB granules, write-back walks of + // inner-shareable tables; the physical size is what the CPU has. + uint64_t mmfr0; + asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0)); + uint64_t ips = mmfr0 & 0xF; + if (ips > 5) ips = 5; // 52-bit output needs LPA2 descriptors + uint64_t tcr = 16ULL // T0SZ + | 1ULL << 8 | 1ULL << 10 // IRGN0, ORGN0: write-back + | 3ULL << 12 // SH0: inner shareable + | 0ULL << 14 // TG0: 4 KiB + | 16ULL << 16 // T1SZ + | 1ULL << 24 | 1ULL << 26 // IRGN1, ORGN1 + | 3ULL << 28 // SH1 + | 2ULL << 30 // TG1: 4 KiB + | ips << 32; // IPS + + // The new tables map the code running here at the same address, so + // the switch is a register write followed by a full invalidate. + asm volatile( + "dsb ish\n\t" + "msr mair_el1, %0\n\t" + "msr tcr_el1, %1\n\t" + "isb\n\t" + "msr ttbr1_el1, %2\n\t" + "msr ttbr0_el1, %3\n\t" + "isb\n\t" + "tlbi vmalle1is\n\t" + "dsb ish\n\t" + "isb" + :: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(rootPhys) : "memory"); + + Kt::KernelLogStream(Kt::OK, "VMM") << "Switched to kernel translation tables (HHDM limit 0x" + << base::hex << g_hhdmLimit << ")"; } - void Paging::MapMMIO(std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::MapMMIO not implemented"); + static void MapKernelPage(uint64_t root, uint64_t phys, uint64_t va, uint64_t attr, const char* what) { + if (va % 0x1000 != 0 || phys % 0x1000 != 0) { + Panic(what, nullptr); + } + volatile Descriptor* entry = WalkCreate(KernelRootFor(root, va), va); + if (entry == nullptr) { + Panic("OOM in Paging (kernel page table allocation)", nullptr); + } + SetLeaf(entry, KernelLeaf(phys, attr), va); } - void Paging::MapWC(std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::MapWC not implemented"); + void Paging::Map(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { + MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormal, + "Value that isn't page-aligned passed as address to Paging::Map!"); } - void Paging::MapUser(std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::MapUser not implemented"); + void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { + MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrDevice, + "Value that isn't page-aligned passed as address to Paging::MapMMIO!"); } - std::uint64_t Paging::GetPhysAddr(std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::GetPhysAddr not implemented"); + void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { + MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormalNc, + "Value that isn't page-aligned passed as address to Paging::MapWC!"); + } + + void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) { + if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) { + Panic("Value that isn't page-aligned passed as address to Paging::MapUser!", nullptr); + } + volatile Descriptor* entry = WalkCreate(rootPhys, virtualAddress); + if (entry == nullptr) { + Panic("OOM in Paging::MapUser (page table allocation)", nullptr); + } + SetLeaf(entry, UserLeaf(physicalAddress, AttrNormal, true, true), virtualAddress); + } + + std::uint64_t Paging::GetPhysAddr(std::uint64_t root, std::uint64_t virtualAddress) { + if (virtualAddress >= KernelHalfBase) { + root = g_kernelHalfRoot; + } + return Translate(root, virtualAddress); } std::uint64_t Paging::GetPhysAddr(std::uint64_t virtualAddress) { return GetPhysAddr(rootPhys, virtualAddress); } + // The kernel half lives in TTBR1, so a new address space starts empty. std::uint64_t Paging::CreateUserAddressSpace() { - NotImplemented("aarch64: Paging::CreateUserAddressSpace not implemented"); + void* page = Memory::g_pfa->AllocateZeroed(); + if (page == nullptr) return 0; + return Memory::SubHHDM(page); } - bool Paging::MapUserIn(std::uint64_t, std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::MapUserIn not implemented"); + bool Paging::MapUserIn(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) { + return MapUserInPermissions(root, physicalAddress, virtualAddress, true, true); } - bool Paging::MapUserInPermissions(std::uint64_t, std::uint64_t, std::uint64_t, bool, bool) { - NotImplemented("aarch64: Paging::MapUserInPermissions not implemented"); + static bool MapUserLeaf(uint64_t root, uint64_t phys, uint64_t va, Descriptor desc, const char* what) { + pagingLock.Acquire(); + if (va % 0x1000 != 0 || phys % 0x1000 != 0) { + pagingLock.Release(); + Panic(what, nullptr); + } + volatile Descriptor* entry = WalkCreate(root, va); + if (entry == nullptr) { + pagingLock.Release(); + return false; + } + SetLeaf(entry, desc, va); + pagingLock.Release(); + return true; } - bool Paging::ProtectUserIn(std::uint64_t, std::uint64_t, bool, bool) { - NotImplemented("aarch64: Paging::ProtectUserIn not implemented"); + bool Paging::MapUserInPermissions(std::uint64_t root, std::uint64_t physicalAddress, + std::uint64_t virtualAddress, bool writable, bool executable) { + return MapUserLeaf(root, physicalAddress, virtualAddress, + UserLeaf(physicalAddress, AttrNormal, writable, executable), + "Non-aligned address in Paging::MapUserIn!"); } - bool Paging::MapUserInWC(std::uint64_t, std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::MapUserInWC not implemented"); + bool Paging::MapUserInWC(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) { + return MapUserLeaf(root, physicalAddress, virtualAddress, + UserLeaf(physicalAddress, AttrNormalNc, true, false), + "Non-aligned address in Paging::MapUserInWC!"); } - void Paging::UnmapUserIn(std::uint64_t, std::uint64_t) { - NotImplemented("aarch64: Paging::UnmapUserIn not implemented"); + // Permission changes need no break-before-make, only the TLB flush. + bool Paging::ProtectUserIn(std::uint64_t root, std::uint64_t virtualAddress, + bool writable, bool executable) { + if ((virtualAddress & 0xFFFULL) != 0) return false; + pagingLock.Acquire(); + volatile Descriptor* entry = WalkExisting(root, virtualAddress); + if (entry == nullptr || !IsUserLeaf(*entry)) { + pagingLock.Release(); + return false; + } + + Descriptor desc = *entry; + if (writable) desc &= ~DescApReadOnly; + else desc |= DescApReadOnly; + if (executable) desc &= ~DescUxn; + else desc |= DescUxn; + *entry = desc; + Arch::FlushTlbPage(virtualAddress); + pagingLock.Release(); + return true; } - void Paging::FreeUserHalf(std::uint64_t) { - NotImplemented("aarch64: Paging::FreeUserHalf not implemented"); + void Paging::UnmapUserIn(std::uint64_t root, std::uint64_t virtualAddress) { + pagingLock.Acquire(); + volatile Descriptor* entry = WalkExisting(root, virtualAddress); + if (entry != nullptr && (*entry & DescValid)) { + *entry = 0; + Arch::FlushTlbPage(virtualAddress); + } + pagingLock.Release(); } - bool Paging::IsUserRangeAccessible(std::uint64_t, std::uint64_t, std::uint64_t, bool) { - NotImplemented("aarch64: Paging::IsUserRangeAccessible not implemented"); + void Paging::FreeUserHalf(std::uint64_t root) { + // User space is the lower 128 TiB: level-0 entries 0-255. + for (int i0 = 0; i0 < 256; i0++) { + Descriptor d0 = TableAt(root)[i0]; + if ((d0 & DescValid) == 0) continue; + uint64_t l1 = d0 & DescAddrMask; + + for (int i1 = 0; i1 < 512; i1++) { + Descriptor d1 = TableAt(l1)[i1]; + if ((d1 & DescValid) == 0) continue; + uint64_t l2 = d1 & DescAddrMask; + + for (int i2 = 0; i2 < 512; i2++) { + Descriptor d2 = TableAt(l2)[i2]; + if ((d2 & DescValid) == 0) continue; + uint64_t l3 = d2 & DescAddrMask; + + for (int i3 = 0; i3 < 512; i3++) { + Descriptor leaf = TableAt(l3)[i3]; + if ((leaf & DescValid) == 0) continue; + + // Write-combining/device pages are not the process's. + if (((leaf & DescAttrMask) >> DescAttrShift) != AttrNormal) continue; + + uint64_t pagePhys = leaf & DescAddrMask; + if (pagePhys != 0) { + Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys)); + } + } + + Memory::g_pfa->Free((void*)Memory::HHDM(l3)); + } + + Memory::g_pfa->Free((void*)Memory::HHDM(l2)); + } + + Memory::g_pfa->Free((void*)Memory::HHDM(l1)); + } } - void Paging::MapEfiRuntime(const montauk::boot::EfiInfo&) { - NotImplemented("aarch64: Paging::MapEfiRuntime not implemented"); + bool Paging::IsUserRangeAccessible(std::uint64_t root, std::uint64_t virtualAddress, + std::uint64_t size, bool requireWrite) { + if (size == 0) return true; + if (virtualAddress >= UserSpaceEnd) return false; + if (virtualAddress + size < virtualAddress) return false; + if (virtualAddress + size > UserSpaceEnd) return false; + + uint64_t page = virtualAddress & ~0xFFFULL; + uint64_t lastPage = (virtualAddress + size - 1) & ~0xFFFULL; + for (;;) { + volatile Descriptor* entry = WalkExisting(root, page); + if (entry == nullptr) return false; + Descriptor desc = *entry; + if (!IsUserLeaf(desc)) return false; + if (requireWrite && (desc & DescApReadOnly)) return false; + if (page == lastPage) break; + page += 0x1000; + } + return true; } } diff --git a/kernel/src/Arch/aarch64/Platform.cpp b/kernel/src/Arch/aarch64/Platform.cpp index 345ca9b..d56cb14 100644 --- a/kernel/src/Arch/aarch64/Platform.cpp +++ b/kernel/src/Arch/aarch64/Platform.cpp @@ -12,12 +12,32 @@ using namespace Kt; +extern "C" char ArchExceptionVectors[]; // Vectors.S + namespace Arch { + // SCTLR_EL1.WXN: writable memory is execute-never + static constexpr uint64_t SctlrWxn = 1ULL << 19; + void EarlyInitialize() { // No per-CPU data yet: Smp::TryGetCurrentCpuData must see null. asm volatile("msr tpidr_el1, xzr"); + // Run on SP_EL1 so exceptions taken from the kernel arrive on the + // stack it is already using (Vectors.S has no SP_EL0 handlers). + asm volatile("mov x9, sp\n\tmsr spsel, #1\n\tmov sp, x9" ::: "x9", "memory"); + + // The vectors need no allocation, so take over from the + // bootloader's right away. + asm volatile("msr vbar_el1, %0\n\tisb" :: "r"(ArchExceptionVectors) : "memory"); + + // Kernel mappings are RWX (Paging.cpp), so writable must not + // imply execute-never. + uint64_t sctlr; + asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + sctlr &= ~SctlrWxn; + asm volatile("msr sctlr_el1, %0\n\tisb" :: "r"(sctlr)); + // CPACR_EL1.FPEN = 0b11: FP/SIMD usable at EL0 and EL1 (the kernel // itself is built -mgeneral-regs-only; Context.S saves user state). uint64_t cpacr; @@ -26,10 +46,10 @@ namespace Arch { asm volatile("msr cpacr_el1, %0\n\tisb" :: "r"(cpacr)); } - // TODO(aarch64): install the VBAR_EL1 vector table (EL1/EL0 synchronous - // exceptions -> panic / SyscallDispatch, IRQ -> GIC -> ArchIrqDispatch). + // Installed by EarlyInitialize. void InitializeTraps() { - KernelLogStream(WARNING, "Arch") << "aarch64: exception vectors not implemented"; + KernelLogStream(OK, "Arch") << "Installed exception vectors at 0x" << base::hex + << (uint64_t)ArchExceptionVectors; } // MAIR_EL1 attribute indices are chosen together with the page-table diff --git a/kernel/src/Arch/aarch64/SyscallFrame.hpp b/kernel/src/Arch/aarch64/SyscallFrame.hpp index d9c1940..651f577 100644 --- a/kernel/src/Arch/aarch64/SyscallFrame.hpp +++ b/kernel/src/Arch/aarch64/SyscallFrame.hpp @@ -9,13 +9,17 @@ namespace Arch { - // Saved by the EL0 synchronous-exception vector for `svc #0`: - // x0-x5 carry the arguments and x8 the system-call number. + // The exception frame Vectors.S saves for `svc #0` (the same layout as + // Arch::PanicFrame): x0-x5 carry the arguments and x8 the system-call + // number; the result is written back to x0. struct SyscallFrame { uint64_t arg1, arg2, arg3, arg4, arg5, arg6; // x0-x5 + uint64_t x6, x7; uint64_t syscall_nr; // x8 - uint64_t elr, spsr, sp_el0; // user PC, PSTATE, SP - uint64_t x19_x30[12]; // callee-saved + LR + uint64_t x9_x30[22]; + uint64_t sp_el0, elr, spsr, esr, far; }; + static_assert(sizeof(SyscallFrame) == 288, "must match Vectors.S"); + } diff --git a/kernel/src/Arch/aarch64/Vectors.S b/kernel/src/Arch/aarch64/Vectors.S new file mode 100644 index 0000000..c186bed --- /dev/null +++ b/kernel/src/Arch/aarch64/Vectors.S @@ -0,0 +1,201 @@ +/* + Vectors.S + aarch64 exception vector table (VBAR_EL1) + Copyright (c) 2026 Daniel Hammer +*/ + +/* + * Every exception saves the same 288-byte frame on SP_EL1, laid out as + * Arch::PanicFrame (InterruptFrame.hpp) and Arch::SyscallFrame: + * + * 0 x0 .. x30 (x30 at 240) + * 248 interrupted SP (SP_EL0 from EL0, the pre-exception SP_EL1 from EL1) + * 256 ELR_EL1 + * 264 SPSR_EL1 + * 272 ESR_EL1 + * 280 FAR_EL1 + * + * The kernel runs on SP_EL1 (SPSel = 1, see Platform.cpp). From EL0, + * SP_EL1 is the top of the thread's kernel stack: ArchEnterUserMode sets + * it before the first ERET and every return to EL0 restores it. + */ + +#define FRAME_SIZE 288 +#define FRAME_SP 248 +#define FRAME_ELR 256 +#define FRAME_SPSR 264 +#define FRAME_ESR 272 +#define FRAME_FAR 280 + + .text + +.macro SAVE_FRAME el + sub sp, sp, #FRAME_SIZE + stp x0, x1, [sp, #0] + stp x2, x3, [sp, #16] + stp x4, x5, [sp, #32] + stp x6, x7, [sp, #48] + stp x8, x9, [sp, #64] + stp x10, x11, [sp, #80] + stp x12, x13, [sp, #96] + stp x14, x15, [sp, #112] + stp x16, x17, [sp, #128] + stp x18, x19, [sp, #144] + stp x20, x21, [sp, #160] + stp x22, x23, [sp, #176] + stp x24, x25, [sp, #192] + stp x26, x27, [sp, #208] + stp x28, x29, [sp, #224] + .if \el == 0 + mrs x21, sp_el0 + .else + add x21, sp, #FRAME_SIZE + .endif + stp x30, x21, [sp, #240] + mrs x22, elr_el1 + mrs x23, spsr_el1 + stp x22, x23, [sp, #FRAME_ELR] + mrs x24, esr_el1 + mrs x25, far_el1 + stp x24, x25, [sp, #FRAME_ESR] +.endm + +/* Interrupts are masked again before ELR/SPSR are reloaded: an IRQ taken + in between would overwrite them. */ +.macro RESTORE_FRAME el + msr daifset, #2 + ldp x22, x23, [sp, #FRAME_ELR] + msr elr_el1, x22 + msr spsr_el1, x23 + .if \el == 0 + ldr x21, [sp, #FRAME_SP] + msr sp_el0, x21 + .endif + ldp x0, x1, [sp, #0] + ldp x2, x3, [sp, #16] + ldp x4, x5, [sp, #32] + ldp x6, x7, [sp, #48] + ldp x8, x9, [sp, #64] + ldp x10, x11, [sp, #80] + ldp x12, x13, [sp, #96] + ldp x14, x15, [sp, #112] + ldp x16, x17, [sp, #128] + ldp x18, x19, [sp, #144] + ldp x20, x21, [sp, #160] + ldp x22, x23, [sp, #176] + ldp x24, x25, [sp, #192] + ldp x26, x27, [sp, #208] + ldp x28, x29, [sp, #224] + ldr x30, [sp, #240] + add sp, sp, #FRAME_SIZE + eret +.endm + +/* Each vector slot is 128 bytes; the handlers live outside the table. */ +.macro VECTOR target + .balign 0x80 + b \target +.endm + + .balign 0x800 + .global ArchExceptionVectors +ArchExceptionVectors: + /* Current EL with SP_EL0: the kernel never runs on SP_EL0. */ + VECTOR el1t_unexpected + VECTOR el1t_unexpected + VECTOR el1t_unexpected + VECTOR el1t_unexpected + /* Current EL with SP_ELx */ + VECTOR el1h_sync + VECTOR el1h_irq + VECTOR el1h_fiq + VECTOR el1h_serror + /* Lower EL, AArch64 */ + VECTOR el0_sync + VECTOR el0_irq + VECTOR el0_fiq + VECTOR el0_serror + /* Lower EL, AArch32: not supported */ + VECTOR el0_aarch32 + VECTOR el0_aarch32 + VECTOR el0_aarch32 + VECTOR el0_aarch32 + +/* ---- Exceptions from the kernel ---- */ + +el1t_unexpected: + SAVE_FRAME 1 + mov x0, sp + mov x1, #0 + bl ArchUnexpectedException + +el1h_sync: + SAVE_FRAME 1 + mov x0, sp + mov x1, #0 + bl ArchSyncException + RESTORE_FRAME 1 + +el1h_irq: + SAVE_FRAME 1 + mov x0, sp + mov x1, #0 + bl ArchIrqException + RESTORE_FRAME 1 + +el1h_fiq: + SAVE_FRAME 1 + mov x0, sp + mov x1, #1 + bl ArchUnexpectedException + +el1h_serror: + SAVE_FRAME 1 + mov x0, sp + mov x1, #2 + bl ArchUnexpectedException + +/* ---- Exceptions from user mode ---- */ + +el0_sync: + SAVE_FRAME 0 + lsr x9, x24, #26 /* ESR_EL1.EC */ + cmp x9, #0x15 /* SVC from AArch64 */ + b.ne 1f + + msr daifclr, #2 + mov x0, sp + bl SyscallDispatch + str x0, [sp, #0] + bl SchedExitIfKilled + RESTORE_FRAME 0 + +1: mov x0, sp + mov x1, #1 + bl ArchSyncException + RESTORE_FRAME 0 + +el0_irq: + SAVE_FRAME 0 + mov x0, sp + mov x1, #1 + bl ArchIrqException + RESTORE_FRAME 0 + +el0_fiq: + SAVE_FRAME 0 + mov x0, sp + mov x1, #1 + bl ArchUnexpectedException + +el0_serror: + SAVE_FRAME 0 + mov x0, sp + mov x1, #2 + bl ArchUnexpectedException + +el0_aarch32: + SAVE_FRAME 0 + mov x0, sp + mov x1, #3 + bl ArchUnexpectedException diff --git a/kernel/src/Boot/Main.cpp b/kernel/src/Boot/Main.cpp index 1d98a47..a491151 100644 --- a/kernel/src/Boot/Main.cpp +++ b/kernel/src/Boot/Main.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -111,6 +112,8 @@ extern "C" void kmain() { Hal::ACPI g_acpi((Hal::ACPI::XSDP*)Memory::HHDM(boot.rsdpPhysical)); + Drivers::Serial::InitializeConsole(g_acpi.GetXSDT()); + if (g_acpi.GetXSDT() != nullptr) { Hal::AcpiShutdown::Initialize(g_acpi.GetXSDT()); diff --git a/kernel/src/Drivers/Serial/Uart.cpp b/kernel/src/Drivers/Serial/Uart.cpp new file mode 100644 index 0000000..a80a342 --- /dev/null +++ b/kernel/src/Drivers/Serial/Uart.cpp @@ -0,0 +1,181 @@ +/* + * Uart.cpp + * Polled serial console from the ACPI SPCR table + * Copyright (c) 2026 Daniel Hammer +*/ + +#include "Uart.hpp" +#include +#include +#include +#include +#include +#include + +using namespace Kt; + +namespace Drivers::Serial { + + struct GenericAddress { + uint8_t AddressSpaceId; // 0 = system memory, 1 = system I/O + uint8_t RegisterBitWidth; + uint8_t RegisterBitOffset; + uint8_t AccessSize; // 1 = byte, 2 = word, 3 = dword, 4 = qword + uint64_t Address; + } __attribute__((packed)); + + struct Spcr { + Hal::ACPI::CommonSDTHeader Header; + uint8_t InterfaceType; + uint8_t Reserved[3]; + GenericAddress BaseAddress; + uint8_t InterruptType; + uint8_t Irq; + uint32_t GlobalSystemInterrupt; + uint8_t BaudRate; + uint8_t Parity; + uint8_t StopBits; + uint8_t FlowControl; + uint8_t TerminalType; + } __attribute__((packed)); + + // SPCR / DBG2 serial interface subtypes + static constexpr uint8_t Type16550 = 0x00; + static constexpr uint8_t Type16450 = 0x01; + static constexpr uint8_t TypePl011 = 0x03; + static constexpr uint8_t TypeSbsa32 = 0x0D; + static constexpr uint8_t TypeSbsaGeneric = 0x0E; + static constexpr uint8_t TypeBcm2835 = 0x10; + static constexpr uint8_t Type16550Generic = 0x12; + + // PL011: data register, flag register (TX FIFO full) + static constexpr uint64_t Pl011Dr = 0x00; + static constexpr uint64_t Pl011Fr = 0x18; + static constexpr uint32_t Pl011FrTxff = 1u << 5; + + // 16550: transmit holding register, line status register (THR empty) + static constexpr uint64_t Uart16550Thr = 0; + static constexpr uint64_t Uart16550Lsr = 5; + static constexpr uint8_t Uart16550LsrThre = 1u << 5; + + // Bounded so a wedged or absent UART cannot hang the console. + static constexpr int TxSpinLimit = 100000; + + static volatile uint8_t* g_mmio = nullptr; + static uint16_t g_port = 0; + static uint32_t g_stride = 1; + + static inline uint32_t Read32(uint64_t offset) { + return *(volatile uint32_t*)(g_mmio + offset); + } + + static inline void Write32(uint64_t offset, uint32_t value) { + *(volatile uint32_t*)(g_mmio + offset) = value; + } + + static void Pl011Putchar(char c) { + for (int i = 0; i < TxSpinLimit && (Read32(Pl011Fr) & Pl011FrTxff); i++) { + Arch::Pause(); + } + Write32(Pl011Dr, (uint8_t)c); + } + + static uint8_t Mmio16550Read(uint64_t reg) { + if (g_stride == 4) return (uint8_t)Read32(reg * 4); + return g_mmio[reg]; + } + + static void Mmio16550Write(uint64_t reg, uint8_t value) { + if (g_stride == 4) Write32(reg * 4, value); + else g_mmio[reg] = value; + } + + static void Mmio16550Putchar(char c) { + for (int i = 0; i < TxSpinLimit && !(Mmio16550Read(Uart16550Lsr) & Uart16550LsrThre); i++) { + Arch::Pause(); + } + Mmio16550Write(Uart16550Thr, (uint8_t)c); + } + + static void Port16550Putchar(char c) { + for (int i = 0; i < TxSpinLimit && !(Io::In8(g_port + Uart16550Lsr) & Uart16550LsrThre); i++) { + Arch::Pause(); + } + Io::Out8((uint8_t)c, g_port + Uart16550Thr); + } + + static Spcr* FindSpcr(Hal::ACPI::CommonSDTHeader* xsdt) { + uint32_t entryCount = (xsdt->Length - sizeof(Hal::ACPI::CommonSDTHeader)) / 8; + uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(Hal::ACPI::CommonSDTHeader)); + + for (uint32_t i = 0; i < entryCount; i++) { + auto* header = (Hal::ACPI::CommonSDTHeader*)Memory::HHDM(entries[i]); + const char* sig = header->Signature; + if (sig[0] == 'S' && sig[1] == 'P' && sig[2] == 'C' && sig[3] == 'R') { + return (Spcr*)header; + } + } + return nullptr; + } + + static volatile uint8_t* MapRegisters(uint64_t phys) { + uint64_t page = phys & ~0xFFFULL; + Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page)); + return (volatile uint8_t*)Memory::HHDM(phys); + } + + void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt) { + if (xsdt == nullptr) return; + + Spcr* spcr = FindSpcr(xsdt); + if (spcr == nullptr || spcr->Header.Length < sizeof(Spcr)) { + return; + } + + const GenericAddress& reg = spcr->BaseAddress; + if (reg.Address == 0) return; + + SerialSink sink = nullptr; + const char* name = ""; + bool isIo = reg.AddressSpaceId == 1; + + switch (spcr->InterfaceType) { + case TypePl011: + case TypeSbsa32: + case TypeSbsaGeneric: + if (isIo) return; + g_mmio = MapRegisters(reg.Address); + sink = Pl011Putchar; + name = "PL011"; + break; + + case Type16550: + case Type16450: + case Type16550Generic: + case TypeBcm2835: + if (isIo) { + if (!Io::HasPortIo) return; + g_port = (uint16_t)reg.Address; + sink = Port16550Putchar; + } else { + // The BCM2835 mini UART is a 16550 with 32-bit registers. + g_stride = (reg.RegisterBitWidth == 32 || reg.AccessSize == 3 + || spcr->InterfaceType == TypeBcm2835) ? 4 : 1; + g_mmio = MapRegisters(reg.Address); + sink = Mmio16550Putchar; + } + name = spcr->InterfaceType == TypeBcm2835 ? "BCM2835 mini UART" : "16550"; + break; + + default: + KernelLogStream(WARNING, "Serial") << "SPCR interface type 0x" << base::hex + << (uint64_t)spcr->InterfaceType << " not supported"; + return; + } + + AttachSerialConsole(sink); + KernelLogStream(OK, "Serial") << "Console on " << name << " at " << (isIo ? "port " : "") + << "0x" << base::hex << reg.Address; + } + +} diff --git a/kernel/src/Drivers/Serial/Uart.hpp b/kernel/src/Drivers/Serial/Uart.hpp new file mode 100644 index 0000000..45c70f6 --- /dev/null +++ b/kernel/src/Drivers/Serial/Uart.hpp @@ -0,0 +1,18 @@ +/* + * Uart.hpp + * Polled serial console from the ACPI SPCR table + * Copyright (c) 2026 Daniel Hammer +*/ + +#pragma once +#include + +namespace Drivers::Serial { + + // Find the firmware's serial console (SPCR: PL011, SBSA generic UART, + // 16550 or the BCM2835 mini UART) and mirror the kernel console to it. + // Uses the baud rate the firmware configured. Needs the kernel page + // tables (the registers are mapped uncached through the HHDM). + void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt); + +} diff --git a/kernel/src/Pci/Pci.cpp b/kernel/src/Pci/Pci.cpp index 86cbb7f..9282df9 100644 --- a/kernel/src/Pci/Pci.cpp +++ b/kernel/src/Pci/Pci.cpp @@ -187,7 +187,20 @@ namespace Pci { Io::Out32(value, ConfigDataPort); } + // Drivers reach config space through the Legacy* calls below. Without + // an I/O address space (ARM) there is no CF8/CFC mechanism, so they use + // ECAM there, at the access's own width (16/8-bit writes must not + // read-modify-write a neighbouring RW1C register). + static bool EcamCovers(uint8_t bus); + + static bool UseEcamForLegacy(uint8_t bus) { + return !Io::HasPortIo && EcamCovers(bus); + } + uint32_t LegacyRead32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) { + if (UseEcamForLegacy(bus)) { + return EcamRead32(g_ecamBase, bus, device, function, offset & 0xFC); + } g_legacyConfigLock.Acquire(); uint32_t value = LegacyRead32Unlocked(bus, device, function, offset); g_legacyConfigLock.Release(); @@ -205,12 +218,20 @@ namespace Pci { } void LegacyWrite32(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) { + if (UseEcamForLegacy(bus)) { + EcamWrite32(g_ecamBase, bus, device, function, offset & 0xFC, value); + return; + } g_legacyConfigLock.Acquire(); LegacyWrite32Unlocked(bus, device, function, offset, value); g_legacyConfigLock.Release(); } void LegacyWrite16(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint16_t value) { + if (UseEcamForLegacy(bus)) { + EcamWrite16(g_ecamBase, bus, device, function, offset & 0xFE, value); + return; + } g_legacyConfigLock.Acquire(); uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC); int shift = (offset & 2) * 8; @@ -221,6 +242,10 @@ namespace Pci { } void LegacyWrite8(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint8_t value) { + if (UseEcamForLegacy(bus)) { + EcamWrite8(g_ecamBase, bus, device, function, offset, value); + return; + } g_legacyConfigLock.Acquire(); uint32_t tmp = LegacyRead32Unlocked(bus, device, function, offset & 0xFC); int shift = (offset & 3) * 8; diff --git a/kernel/src/Terminal/Terminal.cpp b/kernel/src/Terminal/Terminal.cpp index 9493114..028d952 100644 --- a/kernel/src/Terminal/Terminal.cpp +++ b/kernel/src/Terminal/Terminal.cpp @@ -37,6 +37,14 @@ namespace Kt { // ANSI escape sequence filter state static bool g_ansiEscape = false; + static SerialSink g_serialSink = nullptr; + + static void SerialPutchar(char c) { + if (g_serialSink == nullptr) return; + if (c == '\n') g_serialSink('\r'); + g_serialSink(c); + } + static void RingBufferAppend(char c) { // Strip ANSI escape sequences if (c == '\033') { @@ -209,17 +217,19 @@ namespace Kt { // framebuffer terminal below; storing it in the ring makes file // consumers treat one logical newline as two line breaks. RingBufferAppend(c); + SerialPutchar(c); if (g_suppressKernelLog) { return; } - } - - // Once a graphical app takes over, suppress ALL flanterm writes - // (SYS_PRINT from user processes, etc.) to avoid painting text - // over the GUI framebuffer. - if (g_suppressKernelLog && g_kernelLogDepth == 0) { - return; + } else { + // Once a graphical app takes over, suppress ALL flanterm writes + // (SYS_PRINT from user processes, etc.) to avoid painting text + // over the GUI framebuffer. + if (g_suppressKernelLog) { + return; + } + SerialPutchar(c); } if (c == '\n') { @@ -235,6 +245,18 @@ namespace Kt { } } + void AttachSerialConsole(SerialSink sink) { + g_termLock.Acquire(); + uint64_t start = g_klogCount < KLOG_BUF_SIZE ? 0 : g_klogHead; + for (uint64_t i = 0; i < g_klogCount; i++) { + char c = g_klogBuf[(start + i) % KLOG_BUF_SIZE]; + if (c == '\n') sink('\r'); + sink(c); + } + g_serialSink = sink; + g_termLock.Release(); + } + void SuppressKernelLog() { g_suppressKernelLog = true; } diff --git a/kernel/src/Terminal/Terminal.hpp b/kernel/src/Terminal/Terminal.hpp index 2613dc6..1b37c86 100644 --- a/kernel/src/Terminal/Terminal.hpp +++ b/kernel/src/Terminal/Terminal.hpp @@ -43,6 +43,12 @@ namespace Kt void Putchar(char c); void Print(const char *text); + // Mirror console output to a polled serial port (Drivers/Serial). The + // sink receives the whole kernel log, also once the desktop suppresses + // it on screen; what was logged before attaching is replayed first. + using SerialSink = void (*)(char c); + void AttachSerialConsole(SerialSink sink); + // Terminal mutex - protects flanterm from concurrent CPU access. // Uses Mutex (not Spinlock) to keep interrupts enabled while held, // preventing dropped PS/2 mouse/keyboard bytes during log output. diff --git a/scripts/check-arch-neutral.sh b/scripts/check-arch-neutral.sh index ab9bd6d..dada2c6 100755 --- a/scripts/check-arch-neutral.sh +++ b/scripts/check-arch-neutral.sh @@ -93,8 +93,10 @@ if [ -n "$missing" ]; then fi echo "check-arch-neutral: $ARCH objects compile and every symbol resolves" -if command -v "${KERNEL_LD:-ld.lld}" >/dev/null 2>&1; then - make ARCH="$ARCH" >/dev/null +# Same lookup as kernel/GNUmakefile: a system ld.lld, else rustup's. +lld="${KERNEL_LD:-$(command -v ld.lld || ls "$HOME"/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld 2>/dev/null | head -n 1)}" +if [ -n "$lld" ] && command -v "$lld" >/dev/null 2>&1; then + make ARCH="$ARCH" KERNEL_LD="$lld" >/dev/null echo "check-arch-neutral: linked kernel/bin-$ARCH/kernel" fi