feat: add Broadcom FullMAC Wi-Fi driver

.gitignore

@@ -6,12 +6,21 @@
 archives/isos/
 montaukos.org/downloads/*.iso
 *.hdd
+*.img
+rpi4-firmware/
 qemu.log
 toolchain/local/
 toolchain/build/
 toolchain/src/
 toolchain/sysroot/
+toolchain/sysroot-*/
 programs/bin/
+# Non-x86 userspace builds (programs/mk/arch.mk)
+programs/bin-*/
+programs/obj-*/
+programs/src/*/obj-*/
+programs/lib/*/obj-*/
+programs/lib/*/build-*/
 programs/obj/
 programs/src/*/obj/
 programs/libs/*/obj/

GNUmakefile

@@ -10,6 +10,32 @@ QEMUFLAGS := -smp 4 -m 2G -d int -D qemu.log -no-reboot
 
 override IMAGE_NAME := montauk-$(ARCH)
 
+# KERNEL_ONLY=1 builds a ramdisk holding just the boot files (no
+# userspace), for bringing up a kernel on a new architecture.
+KERNEL_ONLY ?= 0
+
+# programs/ builds x86_64 into bin/, other architectures into bin-<arch>/.
+ifeq ($(ARCH),x86_64)
+    PROGRAMS_BIN := programs/bin
+else
+    PROGRAMS_BIN := programs/bin-$(ARCH)
+endif
+
+override EFI_LOADER_x86_64 := BOOTX64.EFI
+override EFI_LOADER_aarch64 := BOOTAA64.EFI
+override EFI_LOADER_riscv64 := BOOTRISCV64.EFI
+override EFI_LOADER_loongarch64 := BOOTLOONGARCH64.EFI
+
+# The host strip handles x86_64 images; other targets use their toolchain's.
+ifeq ($(ARCH),x86_64)
+    STRIP := strip
+else
+    STRIP := toolchain/local/bin/$(ARCH)-montauk-strip
+endif
+
+# AArch64 UEFI firmware for QEMU (Debian/Ubuntu: qemu-efi-aarch64).
+AARCH64_EFI_FIRMWARE ?= /usr/share/qemu-efi-aarch64/QEMU_EFI.fd
+
 # Toolchain for building the 'limine' executable for the host.
 HOST_CC := cc
 HOST_CFLAGS := -g -O2 -pipe
@@ -51,29 +77,72 @@ run-hdd-x86_64: $(IMAGE_NAME).hdd
 		$(QEMUFLAGS)
 
 .PHONY: run-aarch64
+# The machine mirrors the Raspberry Pi 4/400: Cortex-A72 cores, a GICv2, and
+# keyboard and mouse behind a hub (every USB 2 device on the Pi is: the
+# VL805's one USB 2 port carries an on-board hub). QEMU's hub is full-speed,
+# so the transaction-translator path is only exercised on the real board.
+# The kernel console is mirrored to the PL011 (-serial stdio).
 run-aarch64: $(IMAGE_NAME).iso
 	qemu-system-$(ARCH) \
-		-M virt \
+		-M virt,gic-version=2 \
 		-cpu cortex-a72 \
 		-device ramfb \
 		-device qemu-xhci,id=xhci \
-		-device usb-kbd,bus=xhci.0 \
-		-device usb-mouse,bus=xhci.0 \
-		-bios /usr/share/ovmf/OVMF.fd \
+		-device usb-hub,bus=xhci.0,port=1 \
+		-device usb-kbd,bus=xhci.0,port=1.1 \
+		-device usb-mouse,bus=xhci.0,port=1.2 \
+		-bios $(AARCH64_EFI_FIRMWARE) \
 		-cdrom $(IMAGE_NAME).iso \
+		-serial stdio \
 		$(QEMUFLAGS)
 
 .PHONY: run-hdd-aarch64
 run-hdd-aarch64: $(IMAGE_NAME).hdd
 	qemu-system-$(ARCH) \
-		-M virt \
+		-M virt,gic-version=2 \
 		-cpu cortex-a72 \
 		-device ramfb \
 		-device qemu-xhci,id=xhci \
-		-device usb-kbd,bus=xhci.0 \
-		-device usb-mouse,bus=xhci.0 \
-		-bios /usr/share/ovmf/OVMF.fd \
-		-hda $(IMAGE_NAME).hdd \
+		-device usb-hub,bus=xhci.0,port=1 \
+		-device usb-kbd,bus=xhci.0,port=1.1 \
+		-device usb-mouse,bus=xhci.0,port=1.2 \
+		-bios $(AARCH64_EFI_FIRMWARE) \
+		-drive if=none,id=hd0,format=raw,file=$(IMAGE_NAME).hdd \
+		-device virtio-blk-pci,drive=hd0 \
+		-serial stdio \
+		$(QEMUFLAGS)
+
+# The SD card image on QEMU, with the root read through the SDHCI driver:
+# the card goes on sdhci-pci as a padded copy (QEMU wants a power-of-two card;
+# RPI4_QEMU_CARD_MIB, default 4 GiB, leaves the first-boot grow room to work
+# with; every run starts from a fresh copy, so every run is a first boot),
+# and since QEMU's UEFI has no SD driver, the card's ESP is also
+# attached alone (the card's first partition, as its own disk) over virtio
+# for the firmware to boot Limine from.
+RPI4_QEMU_CARD_MIB ?= 4096
+.PHONY: run-rpi4-sd
+run-rpi4-sd: montauk-rpi4.img
+	cp --sparse=always montauk-rpi4.img rpi4-sd-qemu.img
+	size=$$(stat -c %s rpi4-sd-qemu.img); p=$$(($(RPI4_QEMU_CARD_MIB) * 1048576)); \
+		while [ $$p -lt $$size ]; do p=$$((p * 2)); done; truncate -s $$p rpi4-sd-qemu.img
+	end=$$(PATH=$$PATH:/usr/sbin:/sbin sfdisk -d montauk-rpi4.img | awk -F'[=,]' '/montauk-rpi4.img1/ {print $$2 + $$4}'); \
+		dd if=montauk-rpi4.img of=rpi4-esp-qemu.img bs=512 count=$$end status=none
+	PATH=$$PATH:/usr/sbin:/sbin sfdisk --quiet --delete rpi4-esp-qemu.img 2 2>/dev/null || true
+	qemu-system-aarch64 \
+		-M virt,gic-version=2 \
+		-cpu cortex-a72 \
+		-device ramfb \
+		-device qemu-xhci,id=xhci \
+		-device usb-hub,bus=xhci.0,port=1 \
+		-device usb-kbd,bus=xhci.0,port=1.1 \
+		-device usb-mouse,bus=xhci.0,port=1.2 \
+		-bios $(AARCH64_EFI_FIRMWARE) \
+		-drive if=none,id=esp,format=raw,file=rpi4-esp-qemu.img \
+		-device virtio-blk-pci,drive=esp \
+		-device sdhci-pci \
+		-drive if=none,id=sd,format=raw,file=rpi4-sd-qemu.img \
+		-device sd-card,drive=sd \
+		-serial stdio \
 		$(QEMUFLAGS)
 
 .PHONY: run-riscv64
@@ -196,13 +265,23 @@ ports: programs
 	fi
 
 .PHONY: ramdisk
+ifeq ($(KERNEL_ONLY),1)
+ramdisk: limine/limine kernel
+	rm -rf .kernel-only-root
+	mkdir -p .kernel-only-root/boot/limine
+	cp -v limine/$(EFI_LOADER_$(ARCH)) limine.conf .kernel-only-root/boot/limine/
+	cp -v kernel/bin-$(ARCH)/kernel .kernel-only-root/boot/
+	./scripts/mkramdisk.sh .kernel-only-root ramdisk.tar
+	rm -rf .kernel-only-root
+else
 ramdisk: limine/limine kernel programs
-	mkdir -p programs/bin/boot/limine
-	cp -v limine/BOOTX64.EFI programs/bin/boot/limine/
-	cp -v limine.conf programs/bin/boot/limine/
-	cp -v kernel/bin-$(ARCH)/kernel programs/bin/boot/
-	rm -f programs/bin/boot/ramdisk.tar
-	./scripts/mkramdisk.sh programs/bin ramdisk.tar
+	mkdir -p $(PROGRAMS_BIN)/boot/limine
+	cp -v limine/$(EFI_LOADER_$(ARCH)) $(PROGRAMS_BIN)/boot/limine/
+	cp -v limine.conf $(PROGRAMS_BIN)/boot/limine/
+	cp -v kernel/bin-$(ARCH)/kernel $(PROGRAMS_BIN)/boot/
+	rm -f $(PROGRAMS_BIN)/boot/ramdisk.tar
+	./scripts/mkramdisk.sh $(PROGRAMS_BIN) ramdisk.tar
+endif
 
 $(IMAGE_NAME).iso: limine/limine kernel ramdisk
 	rm -rf iso_root
@@ -261,8 +340,8 @@ release: limine/limine kernel ramdisk
 	rm -rf iso_root .release-tmp
 	mkdir -p iso_root/boot
 	cp kernel/bin-$(ARCH)/kernel iso_root/boot/
-	strip iso_root/boot/kernel
-	cp -r programs/bin .release-tmp
+	$(STRIP) iso_root/boot/kernel
+	cp -r $(PROGRAMS_BIN) .release-tmp
 # A release ISO ships only in-tree apps, so drop any out-of-tree port bundles
 # that happen to be staged in programs/bin/apps (each carries a .port marker).
 # Pass RELEASE_WITH_PORTS=1 to deliberately cut a ports-included image.
@@ -286,11 +365,11 @@ ifeq ($(RELEASE_WITHOUT_SDK),1)
 	       .release-tmp/sdk/libexec .release-tmp/sdk/x86_64-montauk
 endif
 	rm -f $(addprefix .release-tmp/os/,$(RELEASE_EXCLUDE_PROGRAMS))
-	find .release-tmp -name '*.elf' -exec strip {} +
+	find .release-tmp -name '*.elf' -exec $(STRIP) {} +
 # The ramdisk carries its own copy of the kernel at 0:/boot/kernel -- that is
 # the image the Installer lays down on the target disk -- and it is not named
 # *.elf, so the find above misses it. Unstripped it is ~6 MB against ~0.7 MB.
-	strip .release-tmp/boot/kernel
+	$(STRIP) .release-tmp/boot/kernel
 	./scripts/mkramdisk.sh .release-tmp iso_root/boot/ramdisk.tar
 	rm -rf .release-tmp
 	mkdir -p iso_root/boot/limine
@@ -336,6 +415,26 @@ ifeq ($(ARCH),loongarch64)
 endif
 	rm -rf iso_root
 
+# Raspberry Pi 4/400 SD card image (UEFI firmware + Limine + MontaukOS).
+# ROOTFS=ext2 (default) puts the system on an ext2 partition read through the
+# kernel's SD driver; ROOTFS=ramdisk has Limine load ramdisk.tar instead.
+# Phony so switching ROOTFS always rebuilds the image.
+ROOTFS ?= ext2
+.PHONY: rpi4-sd montauk-rpi4.img
+rpi4-sd: montauk-rpi4.img
+
+montauk-rpi4.img: limine/limine kernel ramdisk
+ifneq ($(ARCH),aarch64)
+	$(error rpi4-sd needs ARCH=aarch64)
+endif
+	./scripts/mk-rpi4-sd.sh kernel/bin-$(ARCH)/kernel ramdisk.tar $@ $(ROOTFS) $(PROGRAMS_BIN)-rpi
+
+# Graphical imager: write montauk-rpi4.img to an SD card, or update the
+# installation on one in place (scripts/montauk-imager.py, docs/raspberry-pi.md).
+.PHONY: rpi4-imager
+rpi4-imager:
+	./scripts/montauk-imager.py
+
 $(IMAGE_NAME).hdd: limine/limine kernel ramdisk
 	rm -f $(IMAGE_NAME).hdd
 	dd if=/dev/zero bs=1M count=0 seek=64 of=$(IMAGE_NAME).hdd
@@ -367,7 +466,7 @@ endif
 clean:
 	$(MAKE) -C kernel clean
 # 	$(MAKE) -C programs clean
-	rm -rf iso_root .release-tmp $(IMAGE_NAME).iso $(IMAGE_NAME).hdd ramdisk.tar
+	rm -rf iso_root .release-tmp $(IMAGE_NAME).iso $(IMAGE_NAME).hdd montauk-rpi4.img rpi4-sd-qemu.img rpi4-esp-qemu.img ramdisk.tar
 
 .PHONY: distclean
 distclean:

docs/porting.md

@@ -36,7 +36,7 @@ kernel/src/Arch/
   Platform.hpp      boot-sequence hooks, platform device list, reboot
   CpuPower.hpp      frequency scaling / thermal governor (may be no-ops)
   x86_64/           the complete port
-  aarch64/          skeleton port
+  aarch64/          the port in progress (see below)
 ```
 
 Other interfaces a port implements:
@@ -85,25 +85,115 @@ are `Cpu.hpp`, `IoPort.hpp`, `PerCpu.hpp`, `Context.hpp`, `Abi.hpp` and
 - Interrupts are numbered IRQ slots (`Arch::IRQ_COUNT`); slot numbers for
   MSI-capable drivers are fixed in the drivers.
 
-## The aarch64 skeleton
+## The aarch64 port
 
 Real: CPU primitives (DAIF, WFI, `CNTVCT_EL0`, TLBI, TTBR0), per-CPU data via
 `TPIDR_EL1`, the context switch and EL0 entry (`Context.S`), TLS through
 `TPIDR_EL0`, the ELF/TLS/exit-stub ABI (syscall number in x8), the IRQ slot
-dispatch table (`ArchIrqDispatch`), panic-frame printing.
+dispatch table (`ArchIrqDispatch`), panic-frame printing, and:
 
-Not yet written (each is marked `TODO(aarch64)`):
+- Exception vectors (`Vectors.S`, installed in `EarlyInitialize`). Every
+  exception saves one 288-byte frame that is both `Arch::PanicFrame` and
+  `Arch::SyscallFrame`. EL0 `svc` goes to `SyscallDispatch`; other
+  synchronous exceptions go to `ArchSyncException` (`Exceptions.cpp`), which
+  grows user stacks and lazy anonymous memory, kills faulting processes with
+  a crash report, and panics on kernel faults.
+- Translation tables (`Paging.cpp`): 4 KiB granule, 48-bit VAs. The kernel
+  half lives in its own TTBR1 table; `KernelRoot()` is the TTBR0 table of
+  kernel contexts (EFI identity mappings only, so null pointers fault). The
+  HHDM maps only the memory map's RAM and firmware regions, never the holes
+  between them (device space on ARM). MAIR: 0 Normal WB, 1 Normal NC
+  (`MapWC`), 2 Device-nGnRE (`MapMMIO`).
+- GICv2 (`Interrupts.cpp`, from the MADT): IRQ slots map onto INTIDs --
+  `IRQ_TIMER` onto the generic timer's PPI, the MSI slots (24-45) onto the
+  SPIs of a GICv2m MSI frame (`ComposeMsi`), the IPI slots onto SGIs 0/1,
+  and the platform lines (1-23) onto the SPIs `AssignPlatformLine` gives
+  them (see "Interrupt lines" below).
+  Interrupts are completed (EOI) before the handler runs, as on x86; a
+  level-triggered source drops its line first in an ack hook.
+- Generic timer tick (`Timer.cpp`): the EL1 virtual timer at
+  `CNTFRQ_EL0`, its PPI from the GTDT, re-armed from the ack hook.
+- PSCI `SYSTEM_RESET` / `SYSTEM_OFF` (`Arch::Reboot` / `Arch::PowerOff`)
+  through the conduit the FADT's ARM_BOOT_ARCH names. Hardware-reduced
+  ACPI has no PM1 block, so `AcpiShutdown` leaves power-off to the port.
+- `Arch::SyncInstructionCache` after the ELF loader writes code: ARM
+  instruction fetch does not snoop data writes (QEMU hides this; real
+  cores do not). EL0 may maintain caches itself (`SCTLR_EL1.UCI`).
 
-- `Paging.cpp`: translation tables and MAIR (every method panics).
-- Exception vectors (`VBAR_EL1`): synchronous faults, `svc`, IRQs.
-- GIC driver: SPI routing, SGIs for IPIs, MSIs via ITS/GICv2m
-  (`ComposeMsi` returns false, so drivers fall back to INTx/polling).
-- Generic timer tick (`Arch::Timer::InitializeBsp` returns false).
-- Secondary CPU start-up, PSCI reset and suspend.
+- SMP (`Smp.cpp`): Limine wakes each secondary CPU (PSCI `CPU_ON`
+  underneath) on the bootloader's tables; it programs its own system
+  registers and the kernel translation regime (`Bringup.hpp`), its banked
+  GIC CPU interface and timer PPI, then idles until a reschedule SGI.
+  TLB maintenance is broadcast (`tlbi ...is`).
 
-Building it needs `clang` and `ld.lld` (`make kernel ARCH=aarch64`;
-`KERNEL_LD=` points at a different `ld.lld`, e.g. rustup's
-`lib/rustlib/x86_64-unknown-linux-gnu/bin/gcc-ld/ld.lld`).
+The generic kernel is compiled `-fsigned-char` for AArch64: it was written
+for x86's signed plain `char`.
+
+Building it needs `clang` and `ld.lld`; without a system `ld.lld` the
+kernel makefile uses the one rustup ships
+(`~/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld`), and `KERNEL_LD=`
+overrides both. `make run ARCH=aarch64` boots the full image (desktop, apps,
+USB keyboard and mouse) on QEMU `virt` with a GICv2 and Cortex-A72 cores, the
+Raspberry Pi 4/400's configuration; it needs `qemu-system-aarch64` and
+`qemu-efi-aarch64`, and the userspace compiler
+(`./toolchain/build-montauk-toolchain.sh aarch64`). `KERNEL_ONLY=1` builds an
+image without userspace.
+
+The kernel console is mirrored to the serial port the ACPI SPCR table names
+(PL011, SBSA UART, 16550 or the BCM2835 mini UART; `Drivers/Serial/Uart.cpp`),
+including the log written before the port was found. On QEMU that is
+`-serial stdio`.
+
+## Interrupt lines
+
+Drivers for PCI functions without MSI read the Interrupt Line register and
+pass it to `Arch::EnableLegacyIrq`. On x86 the firmware fills that register
+with the function's I/O APIC input (`Arch::PlatformLinesAreFixed()`). On
+other platforms `Pci::AssignInterruptLines` (`Pci/PciAcpi.cpp`) fills it
+instead: it finds each pin's global system interrupt in the root bridge's
+`_PRT` (swizzling through PCI-to-PCI bridges, resolving PNP0C0F link
+devices through `_CRS`), binds the GSI to a platform line with
+`Hal::GsiRouting::LineForGsi` and writes the line number into the register.
+Devices that ACPI describes directly (platform devices) get their line from
+`LineForGsi` too.
+
+## DMA
+
+`Memory/Dma.hpp` serves devices whose DMA does not snoop the CPU caches
+(ACPI `_CCA = 0`): pages for rings and descriptors come uncached, and
+`SyncForDevice` / `SyncForCpu` clean and invalidate streaming buffers
+around a transfer (`Arch::CleanDataCache` / `InvalidateDataCache`). For
+coherent devices every call is the plain allocation or a no-op. The xHCI
+driver uses it; it also orders its TRB writes with `Arch::DmaWriteFence`
+(the cycle bit last, a control TD's Setup TRB after its Data and Status
+TRBs) and its event reads with `Arch::DmaReadFence`. Those fences cost
+nothing on x86 and are required on ARM, where stores to normal memory and
+device registers may otherwise be observed out of order. The other PCI
+drivers (AHCI, NVMe, e1000, HDA) do not have them yet: they work on QEMU,
+which does not reorder, but need them before they meet real ARM hardware.
+
+## Raspberry Pi 4 / 400
+
+See `docs/raspberry-pi.md`: `make ARCH=aarch64 rpi4-sd`, the firmware it
+expects and how the Pi's hardware is reached.
+
+## Userspace
+
+The programs are architecture-neutral C/C++ built by `<arch>-montauk-gcc`
+(see `toolchain/README.md`). `programs/mk/arch.mk` holds everything
+architecture-specific in the build: the toolchain prefix, FP/SIMD flag sets
+(`ARCH_FP_FLAGS` for GUI apps, `ARCH_NOFP_FLAGS` for system programs) and the
+output layout. x86_64 keeps `bin/`, `obj/` and libraries next to their
+sources; other architectures build beside them into `bin-<arch>/`,
+`obj-<arch>/` and `lib/<name>/build-<arch>/`, so both can be built from one
+tree. The few architecture-specific sources are the raw system-call
+wrappers (`#if defined(__x86_64__)` / `__aarch64__` blocks in
+`montauk/syscall.h`, `libc/montauk.h`, `lib/libc/libc.c` and
+`lib/libc/crt/crt1.c`), `lib/libc/arch/setjmp-<arch>.S` and the scanout
+store fence in `gui/framebuffer.hpp`.
+
+Not built for AArch64: tcc (it generates x86 code), the on-OS SDK and the
+Rust programs (their `x86_64-unknown-montauk` target).
 
 ## Adding another architecture
 

docs/raspberry-pi.md

@@ -0,0 +1,344 @@
+# Raspberry Pi 4 / 400
+
+MontaukOS runs on the Raspberry Pi 4 and 400 through the Pi Firmware Task
+Force's UEFI firmware ([pftf/RPi4](https://github.com/pftf/RPi4), an EDK2
+build) in ACPI mode, booting Limine like any other UEFI machine.
+
+## Building and writing the SD card
+
+```bash
+./toolchain/build-montauk-toolchain.sh aarch64   # once
+make ARCH=aarch64 rpi4-sd                        # -> montauk-rpi4.img
+sudo dd if=montauk-rpi4.img of=/dev/sdX bs=4M conv=fsync
+```
+
+Or use the imager, which also updates a card in place:
+
+```bash
+make rpi4-imager                                 # = ./scripts/montauk-imager.py
+```
+
+### The imager
+
+`scripts/montauk-imager.py` is a Tk window (`python3-tk`) that lists the SD
+cards and USB drives attached (never the host's own disks) and what is on
+them, and either writes a fresh image or updates the MontaukOS on a card. It
+runs as the user and does the writing by starting itself again through
+`pkexec` (it asks for the password each time). The same operations work from
+a terminal: `list`, and as root `flash`, `update` and `expand`
+(`--help` for the options). Images can be raw or `.gz`, `.xz` or `.zip`
+compressed; **Build** runs `make ARCH=aarch64 rpi4-sd`.
+
+- **Write a fresh image** erases the card, writes the image, clears the last
+  MiB of the card (a backup GPT from an earlier OS) and reads the card back to
+  verify it. *Expand* grows the root to fill the card right away instead of
+  leaving it to `firstboot.elf`.
+- **Update** leaves the partition table alone and finds both partitions by
+  label, so a root that has grown to fill the card (or was resized by hand)
+  stays as it is. On the ESP every file is written except `RPI_EFI.fd`, which
+  holds the firmware's own settings (*Update the UEFI firmware* replaces it);
+  a changed `config.txt` is kept as `config.txt.bak`. On the root, `os/`,
+  `apps/`, `sdk/`, `boot/` get the image's new and changed files; `config/`
+  and `www/` only get files the card lacks, `users/` and `state/` are not
+  touched. `init.toml` becomes the image's, keeping each service's `enabled`
+  from the card and appending services only the card has; `capabilities.toml`
+  becomes the image's plus grants only the card has (old copies: `.old`).
+  `os/firstboot.elf` is only written if the card still has it, so a grown
+  card does not grow again. *Remove system files the new image no longer
+  ships* prunes `os/`, `apps/`, `sdk/` and `boot/` (keeping `os/btkeys.bin`,
+  the Bluetooth bonds). Unchanged files are skipped and each file is
+  replaced by a rename; the volume is checked with `e2fsck` before and after,
+  and its features against what `Fs/Ext2.cpp` mounts.
+- **Expand** (a host-side first boot) grows the root partition to the end of
+  the card, at most 1 TiB, and the volume with `resize2fs`; `firstboot.elf`
+  then finds nothing to do and deletes itself.
+
+While it works, a runtime udev rule keeps the desktop from automounting the
+card's partitions (Linux writes to an ext2 volume as soon as it mounts it).
+
+`scripts/mk-rpi4-sd.sh` downloads the firmware release once into
+`rpi4-firmware/` (`PFTF_VERSION=v1.53` by default) and builds an MBR image:
+
+| Partition | Type | Contents |
+|---|---|---|
+| 1 | 0xEF, FAT32 | the firmware files, Limine (`EFI/BOOT/BOOTAA64.EFI`), `boot/kernel` |
+| 2 | 0x83, ext2, label `MONTAUKROOT` | the system tree (the contents of `ramdisk.tar`) |
+
+Limine loads only the kernel; the `limine.conf` on the card has no ramdisk
+module. Finding no ramdisk, the kernel mounts the partition labelled
+`MONTAUKROOT` as `0:/` (`Fs/Boot.cpp`), read through the SD driver
+(`Drivers/Storage/Sdhci`), and the ESP after it. Files load as they are
+used instead of the firmware reading the whole 80 MB tree before boot.
+The card can also boot from a USB card reader (or the image from a USB
+stick): the root then comes through `Drivers/USB/MassStorage.cpp`. A USB
+root can register after the boot scan (a SuperSpeed link still training, a
+reader still bringing its card up), so a boot that finds no `MONTAUKROOT`
+waits up to 10 s for one, servicing USB hot-plug meanwhile.
+Unlike the ramdisk, the root is persistent: changes to `0:/` survive a
+reboot, so shut down from the desktop rather than pulling power.
+The image is sized to its contents (`ROOT_FREE_MIB`, default 64, sets the
+free space on the root) and grows to fill the card on first boot.
+
+### First boot: growing the root
+
+Only the SD image ships `0:/os/firstboot.elf` (`programs/src/firstboot`,
+built into `programs/bin-aarch64-rpi/`, which the script lays over the
+system tree), and the script adds its init service, ahead of all others,
+and its capability grant (`storage_admin`, `system_image`) to the image's
+`init.toml` and `capabilities.toml`. On first boot it shows a status card
+in the login screen's style while it calls `SYS_FS_GROW` on drive 0 in a
+loop:
+
+1. The kernel grows the MBR entry of the root partition as far as the
+   filesystem can use (to the end of the card, or 1 TiB), on a 1 MiB
+   boundary (`Gpt::ResizeMbrPartition`).
+2. The ext2 driver grows the mounted volume (`Fs/Ext2.cpp`, "Online
+   grow"): each call writes the bitmaps and zeroed inode tables of a few
+   new block groups past the old end, then the last call commits the
+   bitmaps, descriptor table and superblock. The descriptor table grows
+   into the blocks `mke2fs -O resize_inode -E resize=2^28` reserved; the
+   leftover reservation is freed and `resize_inode` cleared, so the volume
+   grows once.
+
+The new groups' inode tables are the bulk of the writes (0.1% of the card,
+one inode per 256 KiB), written as long multi-block runs. Once the volume has grown, `firstboot.elf` deletes
+itself and init skips its (optional) service from then on. If it fails it
+stays and the next boot tries again; an interrupted grow is safe, since the
+new groups are only referenced once the superblock is written, and a
+partition larger than its filesystem is valid.
+
+`make ARCH=aarch64 rpi4-sd ROOTFS=ramdisk` builds the old layout instead:
+one FAT32 partition with `boot/ramdisk.tar`, which Limine loads through the
+firmware. It boots slowly but does not depend on the kernel's SD driver.
+
+`make ARCH=aarch64 run-rpi4-sd` boots the ext2 image on QEMU with the card
+on `sdhci-pci`, so the root comes through the same driver. QEMU's UEFI has
+no SD driver, so the card's ESP is also attached on its own over virtio for
+the firmware to load Limine from. The card is padded to
+`RPI4_QEMU_CARD_MIB` (default 4096), and every run starts from a fresh
+copy, so every run is a first boot.
+
+The image's `RPI_EFI.fd` comes with a **MontaukOS** boot entry first in the
+firmware's boot order (`scripts/rpi4-efi-bootorder.py` writes it into the
+release's empty variable store), so the card boots on its own instead of
+falling behind the PXE/HTTP entries the firmware creates. The entry names the
+ESP by the image's MBR disk ID and partition number, so it works from the SD
+slot or a USB drive. A card whose `RPI_EFI.fd` predates it keeps its old
+boot order until it is updated with *Update the UEFI firmware*
+(`--firmware`), which also copies the image's disk ID to the card.
+
+The firmware defaults are what MontaukOS needs; leave them:
+
+- **System Table Mode: ACPI.** The kernel reads ACPI only.
+- **Limit RAM to 3 GB: enabled.** The BCM2711's PCIe can only reach the
+  first 3 GB (its tables say so in `_DMA`). The kernel warns at boot when
+  RAM extends past a device's `_DMA` range.
+
+## Serial console
+
+The kernel console (the whole boot log, including what was printed before
+the port was found) is mirrored to the UART the firmware's SPCR table names.
+The firmware's `config.txt` applies the `miniuart-bt` overlay, which puts the
+PL011 on GPIO 14/15: header pins 8 (TX) and 10 (RX), ground on pin 6, 115200
+8N1, 3.3 V levels. On the Pi 400 the header is on the back.
+
+## How the hardware is reached
+
+| Part | Source | Driver |
+|---|---|---|
+| 4x Cortex-A72 | MADT, PSCI (TF-A, SMC) | `Arch/aarch64` SMP via Limine MP |
+| GIC-400 (GICv2) | MADT | `Arch/aarch64/Interrupts.cpp` |
+| Generic timer | GTDT | `Arch/aarch64/Timer.cpp` |
+| Display | UEFI GOP (VideoCore) | Limine framebuffer |
+| Keyboard (built in), USB ports | VL805 xHCI as ACPI `\_SB.SCB0.XHC0` (`PNP0D10`) | `Drivers/USB/Xhci.cpp` platform probe |
+| USB 2 devices (built-in keyboard too) | VIA hub on the VL805's USB 2 root port | `Drivers/USB/Hub.cpp` |
+| Reset, power-off | PSCI | `Arch::Reboot`, `Arch::PowerOff` |
+| Bluetooth (CYW43455 on the 4B, CYW43456 on the 400) | `BCM2EA6` (`\_SB.GDV0.BTH0`) on the mini UART (`BCM2836`) | `Drivers/Bluetooth/Broadcom.cpp`, `Drivers/Serial/BtUart.cpp` |
+| VideoCore firmware mailbox (radio power, clocks, board MAC) | `BCM2849` (`RPIQ`) | `Drivers/Platform/RpiFirmware.cpp` |
+| GPIO pin functions | `BCM2845` (`GPI0`) | `Drivers/Platform/RpiGpio.cpp` |
+| Ethernet (GENET v5 MAC, BCM54213PE PHY) | `BCM6E4E` (`\_SB.ETH0`) | `Drivers/Net/Genet.cpp` |
+| Wi-Fi (the same CYW4345x, SDIO) | Arasan SD controller `BCM2847`, GPIO 34-39 | `Drivers/Net/Wifi/Brcmf*.cpp` |
+
+The VL805 sits behind the BCM2711's PCIe, whose config space is not ECAM
+(the root bridge's `_DSD` says `linux-ecam-quirk-id = "bcm2711"`), so the
+kernel does not enumerate PCI there. The firmware has already brought up the
+link and the VL805 and describes the controller as a platform device: a
+register block at `0x600000000`, GSI 175, and `_CCA = 0`, i.e. its DMA does
+not snoop the CPU caches. `XHC0._INI` (run by the AML interpreter) enables
+the controller's PCI memory decoding and bus mastering. The xHCI driver then
+keeps its rings and contexts in uncached memory, bounces control-transfer
+data through an uncached page, and cleans/invalidates the caller buffers of
+bulk transfers (`Memory/Dma.hpp`). It also watches the event ring from the
+timer tick, so input keeps working if the INTx line never fires.
+
+The VL805 has one USB 2 root port (port 1) and four USB 3 ones. An on-board
+VIA hub (`2109:3431`) sits on the USB 2 port, and the USB 2 lines of every
+socket, and the Pi 400's internal keyboard (`04d9:0007`), hang off it: a
+keyboard or mouse is always a low/full-speed device behind a high-speed hub,
+reached through that hub's transaction translator.
+
+## Bluetooth
+
+The radio's Bluetooth half speaks HCI over a UART (H4) on GPIO 30-33 (CTS,
+RTS, TXD, RXD); its power switch, `BT_ON`, is line 0 of the VideoCore
+firmware's GPIO expander, reached through the mailbox. The firmware tables
+describe it as `BTH0`, whose `UARTSerialBus` names the UART it uses: with
+the `miniuart-bt` overlay the PL011 is the console, so Bluetooth has the mini
+UART. The kernel refuses to take a UART that is the SPCR console.
+
+At boot the driver probe routes the pins (`ALT5` for the mini UART, `ALT3`
+for the PL011), starts the UART (interrupt driven, 16 KiB receive and
+transmit rings) and the H4 transport under the shared HCI layer. Once the
+root is mounted the idle loop finishes, as for USB adapters:
+
+1. power-cycle the controller through `BT_ON`, HCI Reset at 115200 baud;
+2. Read Local Version: the ROM's LMP subversion picks the patch file,
+   `0:/os/firmware/brcm/BCM4345C0.hcd` (0x6119, 4B) or `BCM4345C5.hcd`
+   (0x6606, 400), shipped unmodified from Raspberry Pi OS's bluez-firmware;
+3. raise both ends to the operating speed (vendor `0xFC18`);
+4. Download Minidriver (`0xFC2E`), then every record of the `.hcd` file
+   (Write RAM, then Launch RAM), as Linux's `btbcm` does;
+5. the patched firmware restarts at 115200: reset, raise the speed again.
+
+The mini UART's FIFO is 8 bytes deep. Its hardware RTS is tried first; if
+the controller answers with it on, the link runs at 921600 baud, otherwise
+at 460800 without flow control (what Raspberry Pi OS uses), and at 115200
+if the UART has no interrupt. The baud rate derives from the VPU core
+clock, so `mk-rpi4-sd.sh` pins it (`core_freq=500`, `core_freq_min=500` in
+`config.txt`), as the overlay's documentation requires. A failed fast
+bring-up is retried entirely at 115200.
+
+The CYW43456 reports a chip-wide default address (`43:45:C5:00:1F:AC`);
+the driver replaces it with the board's Ethernet MAC from the firmware.
+`0:/config/bluetooth.toml` still overrides both (Broadcom `0xFC01`).
+
+## Ethernet
+
+The BCM2711's own Gigabit MAC, GENET v5 (a UniMAC behind a DMA engine),
+talks RGMII to a BCM54213PE PHY at MDIO address 1. The firmware describes
+it as `BCM6E4E`: a 64 KiB register block, two interrupts (the driver uses
+the first, which carries the default queue), `_CCA = 0`, and a `_DSD` with
+`phy-mode` (`rgmii-rxid`: the PHY delays the receive clock, the MAC the
+transmit clock) and `brcm,max-dma-burst-size`.
+
+The driver uses only the default queue (ring 16), with all 256 descriptors
+of each direction. The descriptors live in the MAC's own SRAM, so only the
+2 KiB packet buffers need cache maintenance. Received frames start two
+bytes into their buffer (`RBUF_ALIGN_2B`); neither direction uses the
+64-byte status blocks, so there is no checksum offload. The UniMAC accepts
+broadcast and its own address only (the MDF filter).
+
+The interrupt only acknowledges and masks; frames are dispatched from the
+idle bottom half, which also checks the ring itself, so receive works if
+the interrupt never arrives. The PHY is polled once a second (its interrupt
+pin is not described); a link change reprograms the RGMII block and the
+UniMAC's speed and duplex. A link the firmware already negotiated is kept,
+otherwise the PHY is reset and advertises 10/100 and 1000 full duplex.
+
+The MAC address is the board's, read from OTP through the firmware mailbox
+(the one the Bluetooth driver uses too); the address the firmware's own
+driver programmed is the fallback. `dhcp.elf` waits up to 10 s for a wired
+link, since autonegotiation takes about 3 s after the PHY reset.
+
+## Wi-Fi
+
+The radio's WLAN half is an SDIO card on the BCM2711's older SD controller,
+the Arasan-derived EMMC (`BCM2847`, `0x7E300000`); the card slot is on
+EMMC2, so the two never share a bus. Its pins are GPIO 34-39 (ALT3; clock
+without a pull, the rest pulled up, as Linux's `sdio_pins`) and its power
+switch, `WL_ON`, is line 1 of the firmware's GPIO expander. The probe
+claims the controller only while the slot is on EMMC2 (`BRCME88C` present);
+if the tables do not describe the Arasan, its address is taken as EMMC2's
+less 256 KiB.
+
+It is a FullMAC radio: the firmware does the 802.11 MLME itself and the host
+sees Ethernet frames. The driver follows Linux's brcmfmac (ISC license,
+Broadcom; its notice is in `NOTICES.txt`):
+
+| File | Part |
+|---|---|
+| `BrcmfSdio.cpp` | the Arasan as an SDIO host: CMD5/3/7, 4-bit at 25 MHz, CMD52/CMD53, polled PIO, 32-bit register access |
+| `BrcmfChip.cpp` | backplane window (function 1), clocks, the enumeration ROM's core list, firmware + NVRAM download, function 2, the firmware console |
+| `BrcmfProto.cpp` | SDPCM framing and transmit credit, CDC ioctls, BDC data, events, the receive pump, the transmit queue |
+| `BrcmfConnect.cpp` | preinit and CLM, escan, join, keys, link state |
+
+Once the root is mounted the idle loop:
+
+1. routes the pins, switches the EMMC clock on, pulses `WL_ON`, enumerates
+   the card (function 1 blocks of 64 bytes, function 2 of 512);
+2. requests the ALP clock, reads the chip ID: `0x4345` revision 9 is the
+   CYW43456 (Pi 400), any other revision the CYW43455 (Pi 4B), which picks
+   `0:/os/firmware/brcm/brcmfmac43456-sdio.*` or `brcmfmac43455-sdio.*`
+   (shipped unmodified from Raspberry Pi OS's firmware-nonfree);
+3. halts the ARM Cortex-R4, holds the 802.11 core in reset, sizes the RAM
+   from the CR4's banks, writes the firmware at `0x198000` and the NVRAM
+   (stripped to `key=value\0` pairs and a length token) at the top of RAM,
+   reading both back to verify, puts the reset vector at address 0 and
+   releases the CPU;
+4. waits for the HT clock, sends the SDPCM protocol version through the
+   mailbox, enables function 2, the F2 watermark and MES busy signalling of
+   Linux's CYW43455 settings, save/restore if the chip has it, and the card
+   interrupt;
+5. reads `cur_etheraddr` (the NVRAM's shared placeholder address
+   `b8:27:eb:74:f2:6c` is replaced with the board address + 1, as on
+   Raspberry Pi OS), loads the CLM blob (`clmload`, 1400-byte chunks: these
+   firmware builds carry no regulatory data of their own), turns off glom,
+   roaming, ARP/ND offload and power save, enables the events it handles and
+   brings the interface up (`WLC_UP`).
+
+Nothing uses the controller's interrupt: the bus is swept from the idle
+loop (and from the timer tick when the cores never idle), every 10 ms while
+a scan or a connection is active and every 250 ms otherwise, and at once when
+the controller shows the card interrupt. Every sweep reads the SDIO core's
+interrupt status, then every frame waiting in function 2 (a 4-byte header
+read, then the rest). Ioctls wait for transmit credit and their answer by
+sweeping the bus themselves. Frames are delivered with the bus unlocked, and
+transmitting never waits for the bus: data frames go into a 32-frame queue
+that whoever holds the bus, or the next sweep, sends.
+
+Scans are escans (all channels, active, firmware dwell times); each result
+event carries a `brcmf_bss_info_le` whose information elements go into the
+same scan table the Intel driver fills from beacons. A join sets `wpaie`
+(the RSN element our supplicant builds), `wpa_auth`, `auth` and `wsec`, then
+`join` with the BSSID and the chanspec escan reported. The 4-way handshake
+runs in the host's supplicant (`Wpa.cpp`) as for the Intel adapter: EAPOL
+frames are ordinary Ethernet frames here, and the keys go to the firmware
+with `wsec_key` once message 4 has left the transmit queue. The radio does
+AES-CCMP only: GCMP networks are refused, as are WPA3-only (SAE), WEP and
+TKIP ones.
+
+Unanswered ioctls (a leaky bucket of three), a halt reported through
+the mailbox, or repeated bus errors mark the radio failed: the firmware's
+console and any trap record are printed to the kernel log (`WiFi-FW`), and
+the facade's recovery powers the radio off and repeats the bring-up (at
+most three times).
+
+`tests/wifi/run.sh` drives `BrcmfProto.cpp` and `BrcmfConnect.cpp` with the
+real supplicant against a simulated firmware: preinit and the CLM chunks,
+escan parsing, open and WPA2 joins (an independent authenticator checks
+messages 2 and 4 and the installed keys), the wrong passphrase, refusals,
+join timeouts and a firmware that stops answering.
+
+## Not supported yet
+
+- SD high-speed mode: the driver transfers by polled PIO at 25 MHz on a
+  4-bit bus (multi-block CMD18/CMD25 with Auto CMD12), at most 12.5 MB/s;
+  switching the card to 50 MHz (CMD6) would double that.
+- Wi-Fi beyond 802.11 station mode on one interface: no access point mode,
+  no SDIO high speed (50 MHz), no glom, no firmware supplicant.
+- Audio (HDMI via VideoCore, the 3.5 mm jack via PWM).
+
+## Status
+
+Bluetooth works on a Pi 400 (CYW43456), including A2DP music to
+headphones. Ethernet has not run yet: QEMU has no GENET model. Wi-Fi has
+not run on hardware yet either (QEMU has no model of the radio); only its
+protocol and connection layers are tested, in the host harness. It is untested on a Pi 4B (CYW43455), and QEMU has no model of
+the radio.
+
+Everything else above is verified on QEMU only, including the Pi-specific path:
+the firmware's own XHC0 table, retargeted at QEMU's xHCI and loaded with the
+`bcm2711` quirk on QEMU's root bridge, boots to the desktop with non-coherent
+DMA. Real hardware has not been tried yet. The ext2 root on the SD card boots on QEMU (`run-rpi4-sd`); the
+first-boot grow has been tested only against the kernel's ext2 code in a
+host harness (images grown, then checked with `e2fsck`).

kernel/GNUmakefile

@@ -153,8 +153,10 @@ ifeq ($(ARCH),aarch64)
         override CXX += \
             -target aarch64-unknown-none
     endif
+    # Plain char is signed on x86; the generic kernel is written for that.
     override SHARED_FLAGS += \
-        -mgeneral-regs-only
+        -mgeneral-regs-only \
+        -fsigned-char
     override LDFLAGS += \
         -Wl,-m,aarch64elf
 endif
@@ -291,7 +293,9 @@ cc-runtime-$(ARCH)/cc-runtime.a: GNUmakefile cc-runtime/*
 # so a clang-built non-x86 kernel is linked by ld.lld directly.
 ifneq ($(ARCH),x86_64)
     ifeq ($(CXX_IS_CLANG),1)
-        KERNEL_LD ?= ld.lld
+        # Without a system lld, fall back to the one rustup ships.
+        KERNEL_LD ?= $(firstword $(shell command -v ld.lld) \
+            $(wildcard $(HOME)/.rustup/toolchains/*/lib/rustlib/*/bin/gcc-ld/ld.lld))
         override LD_EMULATION_aarch64 := aarch64elf
         override LD_EMULATION_riscv64 := elf64lriscv
         override LD_EMULATION_loongarch64 := elf64loongarch

kernel/src/ACPI/AcpiShutdown.cpp

@@ -38,8 +38,24 @@ namespace Hal {
                 return;
             }
 
-            // Map and parse the DSDT to find \_S5_
             auto* dsdt = (void*)Memory::HHDM(fadt.DsdtAddress);
+
+            AML::InitializeInterpreter(dsdt, xsdt);
+
+            if (AML::GetInterpreter().IsInitialized()) {
+                AcpiDevices::DeviceList devices{};
+                AcpiDevices::EnumerateAll(devices);
+            }
+
+            AcpiSleep::Initialize(xsdt);
+
+            // Hardware-reduced ACPI (ARM) has no PM1 registers: the
+            // architecture powers off instead (Arch::PowerOff).
+            if ((fadt.Flags & FADT::FlagHardwareReduced) || fadt.PM1aControlBlock == 0) {
+                KernelLogStream(INFO, "ACPI") << "No PM1 control block - ACPI S5 shutdown unavailable";
+                return;
+            }
+
             AML::S5Object s5 = AML::FindS5(dsdt);
 
             if (!s5.Valid) {
@@ -55,23 +71,6 @@ namespace Hal {
 
             KernelLogStream(OK, "ACPI") << "ACPI shutdown initialized (PM1a=" << base::hex
                 << (uint64_t)g_pm1aControlBlock << " SLP_TYPa=" << (uint64_t)g_slpTypA << ")";
-
-            // Initialize the full AML interpreter with the DSDT.
-            // This loads the entire DSDT namespace, enabling device enumeration,
-            // method evaluation, and field I/O for the rest of the kernel.
-            AML::InitializeInterpreter(dsdt, xsdt);
-
-            // Enumerate ACPI devices now that the namespace is loaded
-            if (AML::GetInterpreter().IsInitialized()) {
-                AcpiDevices::DeviceList devices{};
-                AcpiDevices::EnumerateAll(devices);
-            }
-
-            // Initialize S3 suspend support (reads FACS and \_S3_ from namespace)
-            AcpiSleep::Initialize(xsdt);
-
-            // Note: ACPI event handling (SCI, power button) is initialized
-            // from Main.cpp after the APIC subsystem is ready.
         }
 
         bool IsAvailable() {

kernel/src/ACPI/FADT.cpp

@@ -107,6 +107,8 @@ namespace Hal {
             result.AcpiEnable       = fadt->AcpiEnable;
             result.AcpiDisable      = fadt->AcpiDisable;
             result.Flags            = fadt->Flags;
+            result.ArmBootArchFlags = fadt->Header.Length >= offsetof(Table, ArmBootArchFlags) + 2
+                                    ? fadt->ArmBootArchFlags : 0;
             result.Valid            = true;
 
             KernelLogStream(INFO, "FADT") << "DSDT at " << base::hex << result.DsdtAddress;

kernel/src/ACPI/FADT.hpp

@@ -69,6 +69,13 @@ namespace Hal {
             uint8_t  X_GPE1Block[12];
         } __attribute__((packed));
 
+        // FADT Flags
+        constexpr uint32_t FlagHardwareReduced = 1u << 20;
+
+        // ARM_BOOT_ARCH flags (ACPI 5.1+)
+        constexpr uint16_t ArmBootPsciCompliant = 1u << 0;
+        constexpr uint16_t ArmBootPsciUseHvc    = 1u << 1;
+
         struct ParsedFADT {
             uint64_t DsdtAddress;
             uint64_t FacsAddress;
@@ -91,6 +98,7 @@ namespace Hal {
             uint8_t  AcpiEnable;
             uint8_t  AcpiDisable;
             uint32_t Flags;
+            uint16_t ArmBootArchFlags;  // 0 when the table predates the field
             bool     Valid;
         };
 

kernel/src/ACPI/GsiRouting.cpp

@@ -0,0 +1,41 @@
+/*
+    * GsiRouting.cpp
+    * Platform interrupt lines for ACPI global system interrupts
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "GsiRouting.hpp"
+#include <Arch/Interrupts.hpp>
+#include <CppLib/Spinlock.hpp>
+
+namespace Hal::GsiRouting {
+
+    static constexpr uint8_t FirstLine = 1;
+    static constexpr uint8_t LineCount = 24;
+
+    static uint32_t g_lineGsi[LineCount] = {};
+    static uint8_t g_nextLine = FirstLine;
+    static kcp::Mutex g_lock;
+
+    uint8_t LineForGsi(uint32_t gsi, bool levelTriggered, bool activeLow) {
+        if (Arch::PlatformLinesAreFixed()) return 0;
+
+        g_lock.Acquire();
+        for (uint8_t line = FirstLine; line < g_nextLine; line++) {
+            if (g_lineGsi[line] == gsi) {
+                g_lock.Release();
+                return line;
+            }
+        }
+
+        uint8_t line = 0;
+        if (g_nextLine < LineCount &&
+            Arch::AssignPlatformLine(g_nextLine, gsi, levelTriggered, activeLow)) {
+            line = g_nextLine++;
+            g_lineGsi[line] = gsi;
+        }
+        g_lock.Release();
+        return line;
+    }
+
+}

kernel/src/ACPI/GsiRouting.hpp

@@ -0,0 +1,19 @@
+/*
+    * GsiRouting.hpp
+    * Platform interrupt lines for ACPI global system interrupts
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+namespace Hal::GsiRouting {
+
+    // The platform line (1-23) that delivers global system interrupt `gsi`,
+    // assigning one on first use (Arch::AssignPlatformLine); every device on
+    // one GSI shares its line. Drivers register their handler on the line
+    // and call Arch::EnableLegacyIrq. Returns 0 where the platform's lines
+    // are wired by the hardware (x86) or when none is left.
+    uint8_t LineForGsi(uint32_t gsi, bool levelTriggered, bool activeLow);
+
+}

kernel/src/Api/Audio.hpp

@@ -11,8 +11,8 @@
 #include <Drivers/Audio/IntelHda.hpp>
 #include <Drivers/Audio/Mixer.hpp>
 #include <Terminal/Terminal.hpp>
-#include <Drivers/USB/Bluetooth/Bluetooth.hpp>
-#include <Drivers/USB/Bluetooth/A2dp.hpp>
+#include <Drivers/Bluetooth/Bluetooth.hpp>
+#include <Drivers/Bluetooth/A2dp.hpp>
 #include <Sched/Scheduler.hpp>
 
 #include "Syscall.hpp"
@@ -55,10 +55,10 @@ namespace montauk::abi {
             return Drivers::Audio::Mixer::SetOutput(
                 (Drivers::Audio::Mixer::Output)value);
         if (cmd == AUDIO_CTL_BT_STATUS) {
-            if (!Drivers::USB::Bluetooth::IsInitialized()) return 0;
-            if (Drivers::USB::Bluetooth::A2dp::IsReady()) return 2;
-            return Drivers::USB::Bluetooth::A2dp::GetState()
-                       == Drivers::USB::Bluetooth::A2dp::State::Idle
+            if (!Drivers::Bluetooth::IsInitialized()) return 0;
+            if (Drivers::Bluetooth::A2dp::IsReady()) return 2;
+            return Drivers::Bluetooth::A2dp::GetState()
+                       == Drivers::Bluetooth::A2dp::State::Idle
                 ? 0 : 1;
         }
 

kernel/src/Api/BluetoothSyscall.hpp

@@ -6,8 +6,8 @@
 
 #pragma once
 #include <cstdint>
-#include <Drivers/USB/Bluetooth/Bluetooth.hpp>
-#include <Drivers/USB/Bluetooth/Hci.hpp>
+#include <Drivers/Bluetooth/Bluetooth.hpp>
+#include <Drivers/Bluetooth/Hci.hpp>
 #include <Libraries/Memory.hpp>
 #include <Sched/Scheduler.hpp>
 #include <Terminal/Terminal.hpp>
@@ -19,14 +19,14 @@ namespace montauk::abi {
 
     static int64_t Sys_BtScan(BtScanResult* buf, int maxCount, uint32_t timeoutMs) {
         if (!buf || maxCount <= 0) return InvalidArgument;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return NoDevice;
+        if (!Drivers::Bluetooth::IsInitialized()) return NoDevice;
 
         // Use a stack buffer for kernel-side results (max 16)
-        Drivers::USB::Bluetooth::Hci::InquiryDevice tmpBuf[16];
+        Drivers::Bluetooth::Hci::InquiryDevice tmpBuf[16];
         int kMax = maxCount;
         if (kMax > 16) kMax = 16;
 
-        int found = Drivers::USB::Bluetooth::Scan(tmpBuf, kMax, timeoutMs);
+        int found = Drivers::Bluetooth::Scan(tmpBuf, kMax, timeoutMs);
         if (found < 0) return found;
 
         // Convert kernel InquiryDevice to userspace BtScanResult
@@ -47,38 +47,38 @@ namespace montauk::abi {
 
     static int64_t Sys_BtConnect(const uint8_t* bdAddr) {
         if (!bdAddr) return InvalidArgument;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return NoDevice;
+        if (!Drivers::Bluetooth::IsInitialized()) return NoDevice;
 
-        return (int64_t)Drivers::USB::Bluetooth::Connect(bdAddr);
+        return (int64_t)Drivers::Bluetooth::Connect(bdAddr);
     }
 
     static int64_t Sys_BtDisconnect(const uint8_t* bdAddr) {
         if (!bdAddr) return InvalidArgument;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return NoDevice;
+        if (!Drivers::Bluetooth::IsInitialized()) return NoDevice;
 
         Kt::KernelLogStream(Kt::INFO, "Power") << "pid " << kcp::dec
             << (uint64_t)Sched::GetCurrentPid()
             << " entered Bluetooth disconnect stage";
-        return (int64_t)Drivers::USB::Bluetooth::Disconnect(bdAddr);
+        return (int64_t)Drivers::Bluetooth::Disconnect(bdAddr);
     }
 
     static int64_t Sys_BtSetAddr(const uint8_t* bdAddr) {
         if (!bdAddr) return InvalidArgument;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return NoDevice;
+        if (!Drivers::Bluetooth::IsInitialized()) return NoDevice;
 
         // bdAddr is a 6-byte buffer, addr[0] = least-significant octet.
-        return Drivers::USB::Bluetooth::SetAddress(bdAddr) ? Success : GenericFailure;
+        return Drivers::Bluetooth::SetAddress(bdAddr) ? Success : GenericFailure;
     }
 
     static int64_t Sys_BtBonds(BtBondInfo* buf, int maxCount) {
         if (!buf || maxCount <= 0) return 0;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return 0;
+        if (!Drivers::Bluetooth::IsInitialized()) return 0;
 
-        Drivers::USB::Bluetooth::Hci::BondInfo tmpBuf[8];
+        Drivers::Bluetooth::Hci::BondInfo tmpBuf[8];
         int kMax = maxCount;
         if (kMax > 8) kMax = 8;
 
-        int count = Drivers::USB::Bluetooth::ListBonded(tmpBuf, kMax);
+        int count = Drivers::Bluetooth::ListBonded(tmpBuf, kMax);
 
         for (int i = 0; i < count; i++) {
             memcpy(buf[i].bdAddr, tmpBuf[i].Addr, 6);
@@ -91,20 +91,20 @@ namespace montauk::abi {
 
     static int64_t Sys_BtForget(const uint8_t* bdAddr) {
         if (!bdAddr) return InvalidArgument;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return NoDevice;
+        if (!Drivers::Bluetooth::IsInitialized()) return NoDevice;
 
-        return (int64_t)Drivers::USB::Bluetooth::ForgetDevice(bdAddr);
+        return (int64_t)Drivers::Bluetooth::ForgetDevice(bdAddr);
     }
 
     static int64_t Sys_BtList(BtDevInfo* buf, int maxCount) {
         if (!buf || maxCount <= 0) return 0;
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return 0;
+        if (!Drivers::Bluetooth::IsInitialized()) return 0;
 
-        Drivers::USB::Bluetooth::Hci::ConnectionInfo tmpBuf[4];
+        Drivers::Bluetooth::Hci::ConnectionInfo tmpBuf[4];
         int kMax = maxCount;
         if (kMax > 4) kMax = 4;
 
-        int count = Drivers::USB::Bluetooth::ListConnected(tmpBuf, kMax);
+        int count = Drivers::Bluetooth::ListConnected(tmpBuf, kMax);
 
         for (int i = 0; i < count; i++) {
             memcpy(buf[i].bdAddr, tmpBuf[i].BdAddr, 6);
@@ -123,14 +123,14 @@ namespace montauk::abi {
 
         memset(buf, 0, sizeof(BtAdapterInfo));
 
-        if (!Drivers::USB::Bluetooth::IsInitialized()) {
+        if (!Drivers::Bluetooth::IsInitialized()) {
             buf->initialized = 0;
             return NoDevice;
         }
 
         buf->initialized = 1;
-        memcpy(buf->bdAddr, Drivers::USB::Bluetooth::GetBdAddr(), 6);
-        buf->scanning = Drivers::USB::Bluetooth::Hci::IsInquiryActive() ? 1 : 0;
+        memcpy(buf->bdAddr, Drivers::Bluetooth::GetBdAddr(), 6);
+        buf->scanning = Drivers::Bluetooth::Hci::IsInquiryActive() ? 1 : 0;
 
         // Copy adapter name
         const char* name = "MontaukOS";

kernel/src/Api/BuildNo.hpp

@@ -12,4 +12,4 @@
 
 #pragma once
 
-#define MONTAUK_BUILD_NUMBER 26
+#define MONTAUK_BUILD_NUMBER 123

kernel/src/Api/Device.hpp

@@ -8,9 +8,10 @@
 #include <cstdint>
 #include <Drivers/USB/Xhci.hpp>
 #include <Drivers/USB/UsbDevice.hpp>
-#include <Drivers/USB/Bluetooth/Bluetooth.hpp>
+#include <Drivers/Bluetooth/Bluetooth.hpp>
 #include <Drivers/Net/E1000.hpp>
 #include <Drivers/Net/E1000E.hpp>
+#include <Drivers/Net/Genet.hpp>
 #include <Drivers/Net/Wifi/Wifi.hpp>
 #include <Drivers/Graphics/IntelGPU.hpp>
 #include <Drivers/Storage/Ahci.hpp>
@@ -122,11 +123,24 @@ namespace montauk::abi {
         if (Drivers::Net::E1000E::IsInitialized()) {
             add(5, "Intel E1000E", "Gigabit Ethernet (82574L)");
         }
+        if (Drivers::Net::Genet::IsInitialized()) {
+            uint32_t speed = Drivers::Net::Genet::GetLinkSpeed();
+            add(5, "Broadcom GENET", speed == 1000 ? "Gigabit Ethernet (1000 Mbit/s)"
+                                   : speed == 100  ? "Gigabit Ethernet (100 Mbit/s)"
+                                   : speed == 10   ? "Gigabit Ethernet (10 Mbit/s)"
+                                   : "Gigabit Ethernet (no link)");
+        }
         if (Drivers::Net::Wifi::IsPresent()) {
             add(5, "Intel Wi-Fi", Drivers::Net::Wifi::IsLinkUp()
                                   ? "802.11 wireless (connected)"
                                   : "802.11 wireless");
         }
+        // An on-board UART Bluetooth controller (Raspberry Pi) is no USB
+        // device, so it is listed here; USB adapters appear above.
+        if (Drivers::Bluetooth::IsInitialized() &&
+            Drivers::Bluetooth::Hci::GetTransport() == Drivers::Bluetooth::Hci::Transport::Uart) {
+            add(5, "Bluetooth Adapter", "Broadcom, HCI over UART");
+        }
 
         // Display (category 6)
         if (Drivers::Graphics::IntelGPU::IsInitialized()) {
@@ -148,7 +162,7 @@ namespace montauk::abi {
             add(9, "Intel HDA", detail);
         }
 
-        if (Drivers::USB::Bluetooth::IsInitialized()) {
+        if (Drivers::Bluetooth::IsInitialized()) {
             add(9, "Bluetooth Audio", "A2DP (SBC)");
         }
 
@@ -248,6 +262,10 @@ namespace montauk::abi {
         case Drivers::Storage::BLOCK_KIND_USB_MSC:
             buf->type = 4;
             break;
+        case Drivers::Storage::BLOCK_KIND_SD:
+            buf->type = 5;
+            buf->rpm = 1;
+            break;
         default:
             break;
         }

kernel/src/Api/Net.hpp

@@ -18,6 +18,7 @@
 #include <Net/NetIf.hpp>
 #include <Drivers/Net/E1000.hpp>
 #include <Drivers/Net/E1000E.hpp>
+#include <Drivers/Net/Genet.hpp>
 #include <Drivers/Net/Wifi/Wifi.hpp>
 
 #include "Syscall.hpp"
@@ -157,6 +158,10 @@ namespace montauk::abi {
             out->rxPackets = Drivers::Net::E1000E::GetRxPacketCount();
             out->txPackets = Drivers::Net::E1000E::GetTxPacketCount();
             CopyNetStatusDriver(out->driver, "Intel e1000e");
+        } else if (Drivers::Net::Genet::IsInitialized()) {
+            out->rxPackets = Drivers::Net::Genet::GetRxPacketCount();
+            out->txPackets = Drivers::Net::Genet::GetTxPacketCount();
+            CopyNetStatusDriver(out->driver, "Broadcom GENET");
         }
     }
 

kernel/src/Api/Power.hpp

@@ -73,6 +73,9 @@ namespace montauk::abi {
             Hal::AcpiShutdown::Shutdown();
         }
 
+        /* Hardware-reduced platforms (PSCI on ARM) */
+        Arch::PowerOff();
+
         /* Fallback: EFI ResetSystem */
         if (Efi::g_ResetSystem) {
             Memory::VMM::ActivateKernelAddressSpace();

kernel/src/Api/Storage.hpp

@@ -163,6 +163,45 @@ namespace montauk::abi {
         return (int64_t)Fs::FsProbe::SyncAndUnmountAll();
     }
 
+    // Grow a mounted volume to fill the free space after its partition: the
+    // MBR entry first, only as far as the filesystem can use, then the
+    // filesystem a slice per call. Interrupted grows resume from the next
+    // call (or boot), since a partition larger than its filesystem is valid.
+    static int64_t Sys_FsGrow(int drive, FsGrowStatus* out, bool apply) {
+        int partIndex = Fs::FsProbe::GetPartitionForDrive(drive);
+        auto* part = Drivers::Storage::Gpt::GetPartition(partIndex);
+        if (part == nullptr) return UnsupportedOp;   // ramdisk, or no such drive
+
+        uint64_t maxSectors = Drivers::Storage::Gpt::GetMbrPartitionMaxSectors(partIndex);
+        if (maxSectors < part->SectorCount) maxSectors = part->SectorCount;
+
+        Fs::Vfs::GrowStatus gs{};
+        int rc = Fs::Vfs::VfsGrow(drive, maxSectors, false, &gs);
+        if (rc == -2) return UnsupportedOp;
+        if (rc < 0) return IoFailure;
+
+        // Leave whatever the filesystem cannot use unpartitioned, with the end
+        // on a 1 MiB boundary like the partitions mk-rpi4-sd.sh makes.
+        uint64_t wanted = (part->StartLba + (gs.targetBytes + 511) / 512 + 2047) / 2048 * 2048
+                        - part->StartLba;
+        if (wanted > maxSectors) wanted = maxSectors;
+
+        if (apply && wanted > part->SectorCount) {
+            if (Drivers::Storage::Gpt::ResizeMbrPartition(partIndex, wanted) != 0) return IoFailure;
+        }
+
+        rc = apply ? Fs::Vfs::VfsGrow(drive, part->SectorCount, true, &gs) : 0;
+        if (rc == -2) return UnsupportedOp;
+        if (rc < 0) return IoFailure;
+
+        out->fsBytes = gs.fsBytes;
+        out->partBytes = part->SectorCount * 512;
+        out->targetBytes = gs.targetBytes;
+        out->stepsDone = gs.stepsDone;
+        out->stepsTotal = gs.stepsTotal;
+        return rc;
+    }
+
     // Format a partition with a filesystem. Returns 0 on success, GenericFailure on error.
     static int64_t Sys_FsFormat(const FsFormatParams* params) {
         if (params == nullptr) return InvalidArgument;

kernel/src/Api/Syscall.cpp

@@ -16,7 +16,7 @@
 #include "Terminal.hpp"   // SYS_PRINT, SYS_PUTCHAR
 #include "Filesystem.hpp" // SYS_OPEN, SYS_READ, SYS_GETSIZE, SYS_CLOSE, SYS_READDIR, SYS_FWRITE, SYS_FCREATE
 #include "Heap.hpp"       // SYS_ALLOC, SYS_FREE
-#include "Time.hpp"       // SYS_GETTICKS, SYS_GETMILLISECONDS, SYS_GETTIME
+#include "Time.hpp"       // SYS_GETTICKS, SYS_GETMILLISECONDS, SYS_GETTIME, SYS_GETUNIXTIME
 #include "Keyboard.hpp"   // SYS_ISKEYAVAILABLE, SYS_GETKEY, SYS_GETCHAR
 #include "Info.hpp"       // SYS_GETINFO
 #include "Graphics.hpp"   // SYS_FBINFO, SYS_FBMAP, SYS_TERMSIZE, SYS_TERMSCALE
@@ -245,6 +245,8 @@ namespace montauk::abi {
                 if (!UserMemory::Writable<DateTime>(frame->arg1)) return BadPointer;
                 Sys_GetTime((DateTime*)frame->arg1);
                 return Success;
+            case SYS_GETUNIXTIME:
+                return Sys_GetUnixTime();
             case SYS_SETUNIXTIME:
                 if (!Sched::Consent::Require(CAP_SET_TIME)) return CapabilityDenied;
                 return Sys_SetUnixTime((int64_t)frame->arg1);
@@ -454,6 +456,10 @@ namespace montauk::abi {
             case SYS_FS_SYNC:
                 if (!Sched::Consent::Require(CAP_STORAGE_ADMIN)) return CapabilityDenied;
                 return Sys_FsSync();
+            case SYS_FS_GROW:
+                if (!Sched::Consent::Require(CAP_STORAGE_ADMIN)) return CapabilityDenied;
+                if (!UserMemory::Range(frame->arg2, sizeof(FsGrowStatus), true)) return BadPointer;
+                return Sys_FsGrow((int)frame->arg1, (FsGrowStatus*)frame->arg2, frame->arg3 != 0);
             case SYS_FSFORMAT:
                 if (!Sched::Consent::Require(CAP_STORAGE_ADMIN)) return CapabilityDenied;
                 if (!UserMemory::Readable<FsFormatParams>(frame->arg1)) return BadPointer;

kernel/src/Api/Syscall.hpp

@@ -400,6 +400,14 @@ namespace montauk::abi {
     static constexpr uint64_t SYS_WIN_DRAG_FORMATS     = 206;  // (serial, TransferFormatInfo*, max) -> count
     static constexpr uint64_t SYS_WIN_DRAG_GET_DATA    = 207;  // (serial, mime, buf, max, uint32_t* len) -> status
 
+    /* Storage.hpp -- grow a mounted volume (and its MBR partition) to fill the
+       free space after it. apply=0 only reports; apply=1 does one bounded slice
+       and returns 1 while more remains, 0 once done. CAP_STORAGE_ADMIN. */
+    static constexpr uint64_t SYS_FS_GROW              = 208;  // (drive, FsGrowStatus*, apply) -> 1 / 0 / status
+
+    /* Time.hpp -- wall clock as UTC unix seconds (SYS_GETTIME is local time). */
+    static constexpr uint64_t SYS_GETUNIXTIME          = 209;  // () -> seconds
+
     /* Kernel-owned process capabilities. User identities may namespace
        per-user resources, but never participate in authorization decisions. */
     static constexpr uint64_t CAP_PROCESS_ADMIN = 1ULL << 0;
@@ -915,7 +923,7 @@ namespace montauk::abi {
 
     struct DiskInfo {
         uint8_t  port;              // block device index
-        uint8_t  type;              // 0=none, 1=SATA, 2=SATAPI, 3=NVMe, 4=USB mass storage
+        uint8_t  type;              // 0=none, 1=SATA, 2=SATAPI, 3=NVMe, 4=USB mass storage, 5=SD card
         uint8_t  sataGen;           // SATA gen (1/2/3)
         uint8_t  _pad0;
         uint64_t sectorCount;       // Total user-addressable sectors
@@ -997,6 +1005,14 @@ namespace montauk::abi {
         char     label[32];       // volume label
     };
 
+    struct FsGrowStatus {
+        uint64_t fsBytes;         // filesystem size now
+        uint64_t partBytes;       // partition size now
+        uint64_t targetBytes;     // filesystem size once grown; == fsBytes if it cannot grow
+        uint32_t stepsDone;       // progress of the grow, out of stepsTotal
+        uint32_t stepsTotal;
+    };
+
     struct ProcInfo {
         int32_t  pid;
         int32_t  parentPid;

kernel/src/Api/Time.hpp

@@ -1,6 +1,6 @@
 /*
     * Time.hpp
-    * SYS_GETTICKS, SYS_GETMILLISECONDS, SYS_GETTIME syscalls
+    * SYS_GETTICKS, SYS_GETMILLISECONDS, SYS_GETTIME, SYS_GETUNIXTIME syscalls
     * Copyright (c) 2026 Daniel Hammer
 */
 
@@ -39,6 +39,10 @@ namespace montauk::abi {
         return (int64_t)Timekeeping::GetTZOffset();
     }
 
+    static int64_t Sys_GetUnixTime() {
+        return Timekeeping::GetUnixTimestamp();
+    }
+
     static int64_t Sys_SetUnixTime(int64_t unixSeconds) {
         return Timekeeping::SetUnixTimestamp(unixSeconds) ? Success : InvalidArgument;
     }

kernel/src/Arch/Cpu.hpp

@@ -36,6 +36,15 @@ namespace Arch {
     // Full read/write memory barrier.
     inline void MemoryFence();
 
+    // Order this CPU's earlier writes to DMA memory (descriptors, buffers)
+    // before a later write the device acts on: the descriptor's valid/cycle
+    // bit, or a doorbell register. x86 keeps stores in order already.
+    inline void DmaWriteFence();
+
+    // Order the read of a device-written valid/cycle/status field before the
+    // reads of the data it guards.
+    inline void DmaReadFence();
+
     // ---- Interrupt state ----
 
     inline void EnableInterrupts();
@@ -79,6 +88,18 @@ namespace Arch {
     // Write back and invalidate all CPU caches (before power state changes).
     inline void WritebackInvalidateCaches();
 
+    // Cache maintenance for devices whose DMA does not snoop the CPU caches
+    // (ACPI _CCA = 0). Clean: write dirty lines back so the device reads
+    // what the CPU wrote. Invalidate: write back and discard, so the CPU
+    // then reads what the device wrote. No-ops where DMA is coherent (x86).
+    inline void CleanDataCache(const void* address, uint64_t size);
+    inline void InvalidateDataCache(const void* address, uint64_t size);
+
+    // Make instructions just written to [address, address+size) (any
+    // mapping of them) visible to instruction fetch on every CPU. A no-op
+    // where instruction fetch snoops data writes (x86).
+    inline void SyncInstructionCache(const void* address, uint64_t size);
+
     // Invalidate this CPU's translation for one virtual page.
     inline void FlushTlbPage(uint64_t virtualAddress);
 

kernel/src/Arch/Interrupts.hpp

@@ -53,6 +53,17 @@ namespace Arch {
     // Unmask platform interrupt line `irq` (already routed at boot).
     void EnableLegacyIrq(uint8_t irq);
 
+    // Whether the platform interrupt lines are wired by the hardware (x86:
+    // ISA IRQs and I/O APIC pins, as the firmware programs them into PCI
+    // Interrupt Line registers). Where they are not (ARM), the PCI layer
+    // assigns lines to PCI INTx pins from ACPI _PRT with AssignPlatformLine.
+    bool PlatformLinesAreFixed();
+
+    // Make platform line `irq` (1-23) deliver global system interrupt `gsi`
+    // (ACPI numbering) with the given trigger mode and polarity, so that
+    // EnableLegacyIrq(irq) unmasks it. Only where !PlatformLinesAreFixed().
+    bool AssignPlatformLine(uint8_t irq, uint32_t gsi, bool levelTriggered, bool activeLow);
+
     // A message-signalled interrupt targeting IRQ slot `irq` on the boot
     // CPU: the device writes `data` to `address` to raise it. Program these
     // into a PCI MSI capability or MSI-X table entry. Returns false when the

kernel/src/Arch/Platform.hpp

@@ -51,4 +51,8 @@ namespace Arch {
     // ResetSystem). Never returns.
     [[noreturn]] void Reboot();
 
+    // Switch the machine off where ACPI S5 does not apply (hardware-reduced
+    // ACPI: ARM's PSCI SYSTEM_OFF). Returns if the platform cannot.
+    void PowerOff();
+
 }

kernel/src/Arch/aarch64/Bringup.hpp

@@ -0,0 +1,22 @@
+/*
+    * Bringup.hpp
+    * aarch64 per-CPU bring-up shared by the boot CPU and secondaries
+    * (private to the port)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+
+namespace Arch {
+
+    // System registers every CPU programs for itself: SP_EL1 as the stack,
+    // the exception vectors, SCTLR_EL1 policy bits, FP/SIMD access.
+    // Platform.cpp.
+    void InitializeCpuRegisters();
+
+    // Load the kernel's translation regime (MAIR, TCR, TTBR1 kernel half,
+    // TTBR0 kernel-context table) on the calling CPU. Paging::Init builds
+    // the tables and calls it on the boot CPU. Paging.cpp.
+    void ActivateKernelTranslation();
+
+}

kernel/src/Arch/aarch64/Cpu.hpp

@@ -26,6 +26,15 @@ namespace Arch {
         asm volatile("dsb sy" ::: "memory");
     }
 
+    // Devices are outer-shareable observers.
+    inline void DmaWriteFence() {
+        asm volatile("dmb oshst" ::: "memory");
+    }
+
+    inline void DmaReadFence() {
+        asm volatile("dmb oshld" ::: "memory");
+    }
+
     inline void EnableInterrupts() {
         asm volatile("msr daifclr, #2" ::: "memory");
     }
@@ -92,9 +101,50 @@ namespace Arch {
         asm volatile("dsb sy\n\tisb" ::: "memory");
     }
 
+    // The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0.
+    inline uint64_t DataCacheLineSize() {
+        uint64_t ctr;
+        asm volatile("mrs %0, ctr_el0" : "=r"(ctr));
+        return 4ULL << ((ctr >> 16) & 0xF);  // DminLine
+    }
+
+    // To the point of coherency: where non-snooping device DMA looks.
+    inline void CleanDataCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = (uint64_t)address & ~(line - 1); p < end; p += line) {
+            asm volatile("dc cvac, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb sy" ::: "memory");
+    }
+
+    // Clean too: a partial line at either end may hold CPU data next to the
+    // buffer, which a plain invalidate would throw away.
+    inline void InvalidateDataCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = (uint64_t)address & ~(line - 1); p < end; p += line) {
+            asm volatile("dc civac, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb sy" ::: "memory");
+    }
+
+    // Clean the data cache to the point of unification, then invalidate
+    // the whole instruction cache: invalidating by address would only
+    // reach the alias it names on an aliasing (VIPT) instruction cache.
+    inline void SyncInstructionCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t start = (uint64_t)address & ~(line - 1);
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = start; p < end; p += line) {
+            asm volatile("dc cvau, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb ish\n\tic ialluis\n\tdsb ish\n\tisb" ::: "memory");
+    }
+
     inline void FlushTlbPage(uint64_t virtualAddress) {
         asm volatile("dsb ishst\n\ttlbi vaae1is, %0\n\tdsb ish\n\tisb"
-                     :: "r"(virtualAddress >> 12) : "memory");
+                     :: "r"((virtualAddress >> 12) & ((1ULL << 44) - 1)) : "memory");
     }
 
     inline void FlushTlbAll() {

kernel/src/Arch/aarch64/Exceptions.cpp

@@ -0,0 +1,184 @@
+/*
+    * Exceptions.cpp
+    * aarch64 synchronous exceptions: faults, user crashes, kernel panics
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include <Arch/aarch64/InterruptFrame.hpp>
+#include <Arch/Cpu.hpp>
+#include <Arch/Smp.hpp>
+#include <Common/Panic.hpp>
+#include <Common/TerminationCodes.hpp>
+#include <CppLib/Stream.hpp>
+#include <Memory/Paging.hpp>
+#include <Sched/Scheduler.hpp>
+#include <Sched/CrashReport.hpp>
+#include <Api/Heap.hpp>
+#include <Timekeeping/Timer.hpp>
+
+using namespace Kt;
+
+namespace {
+
+    // ESR_EL1.EC exception classes
+    constexpr uint64_t EcUnknown         = 0x00;
+    constexpr uint64_t EcIllegalState    = 0x0E;
+    constexpr uint64_t EcInstAbortLower  = 0x20;
+    constexpr uint64_t EcInstAbortSame   = 0x21;
+    constexpr uint64_t EcPcAlignment     = 0x22;
+    constexpr uint64_t EcDataAbortLower  = 0x24;
+    constexpr uint64_t EcDataAbortSame   = 0x25;
+    constexpr uint64_t EcSpAlignment     = 0x26;
+    constexpr uint64_t EcFpException     = 0x2C;
+    constexpr uint64_t EcBreakpointLower = 0x30;
+    constexpr uint64_t EcBrk             = 0x3C;
+
+    constexpr uint64_t UserSpaceEnd = 0x0000800000000000ULL;
+
+    uint64_t ExceptionClass(const Arch::PanicFrame* frame) {
+        return (frame->esr >> 26) & 0x3F;
+    }
+
+    bool IsAbort(uint64_t ec) {
+        return ec == EcInstAbortLower || ec == EcInstAbortSame
+            || ec == EcDataAbortLower || ec == EcDataAbortSame;
+    }
+
+    // Fault status code (ISS[5:0]) of an abort: translation faults mean
+    // the page is not mapped, permission faults that it is.
+    bool IsTranslationFault(uint64_t esr) {
+        uint64_t fsc = esr & 0x3F;
+        return fsc >= 0x04 && fsc <= 0x07;
+    }
+
+    bool IsPermissionFault(uint64_t esr) {
+        uint64_t fsc = esr & 0x3F;
+        return fsc >= 0x0C && fsc <= 0x0F;
+    }
+
+    const char* ExceptionName(uint64_t ec) {
+        switch (ec) {
+            case EcUnknown:         return "Undefined Instruction";
+            case EcIllegalState:    return "Illegal Execution State";
+            case EcInstAbortLower:
+            case EcInstAbortSame:   return "Instruction Abort";
+            case EcPcAlignment:     return "PC Alignment Fault";
+            case EcDataAbortLower:
+            case EcDataAbortSame:   return "Data Abort";
+            case EcSpAlignment:     return "SP Alignment Fault";
+            case EcFpException:     return "Floating-Point Exception";
+            case EcBrk:             return "Breakpoint (BRK)";
+            default:
+                if (ec >= EcBreakpointLower && ec <= 0x35) return "Debug Exception";
+                return "Synchronous Exception";
+        }
+    }
+
+    TerminationCodes TerminationFromClass(uint64_t ec) {
+        if (ec == EcUnknown || ec == EcIllegalState) return TerminatedIllegalInstruction;
+        if (ec == EcBrk || (ec >= EcBreakpointLower && ec <= 0x35)) return TerminatedBreakpoint;
+        if (ec == EcFpException) return TerminatedArithmeticError;
+        return TerminatedAccessViolation;
+    }
+
+    template<size_t N>
+    void CopyName(char (&dst)[N], const char* src) {
+        size_t i = 0;
+        for (; i + 1 < N && src[i]; i++) dst[i] = src[i];
+        dst[i] = '\0';
+    }
+
+    // Kill the faulting process with a crash report. Never returns.
+    [[noreturn]] void TerminateUserProcess(Arch::PanicFrame* frame, uint64_t ec) {
+        // Teardown can wait on devices and other CPUs; keep the clock running.
+        Arch::EnableInterrupts();
+
+        auto* proc = Sched::GetCurrentProcessPtr();
+        const char* name = ExceptionName(ec);
+        KernelLogStream(ERROR, "Exception") << name << " in process \"" << proc->name
+            << "\" (pid " << proc->pid << ", ESR 0x" << base::hex << frame->esr
+            << ", PC 0x" << frame->elr << ") - process terminated";
+
+        CrashReport::Report rep{};
+        rep.valid = true;
+        rep.pid = proc->pid;
+        CopyName(rep.processName, proc->name);
+        rep.exceptionVector = (uint8_t)ec;
+        CopyName(rep.exceptionName, name);
+        rep.instructionPointer = frame->elr;
+        rep.flags = frame->spsr;
+        rep.stackPointer = frame->sp;
+
+        constexpr uint64_t snapshotBytes = CrashReport::StackSnapshotWords * sizeof(uint64_t);
+        if (Memory::VMM::Paging::IsUserRangeAccessible(proc->pageTableRoot, frame->sp,
+                                                       snapshotBytes, false)) {
+            auto* userStack = (const volatile uint64_t*)frame->sp;
+            for (int word = 0; word < CrashReport::StackSnapshotWords; word++)
+                rep.stackWords[word] = userStack[word];
+            rep.stackWordCount = CrashReport::StackSnapshotWords;
+        }
+
+        if (IsAbort(ec)) {
+            rep.faultingAddress = frame->far;
+            rep.pfPresent = IsPermissionFault(frame->esr);
+            rep.pfWrite = ec == EcDataAbortLower && (frame->esr & (1ULL << 6)) != 0;
+            rep.pfUser = 1;
+            rep.pfInstructionFetch = ec == EcInstAbortLower;
+        }
+        rep.timestampTick = Timekeeping::GetTicks();
+
+        CrashReport::AddReport(rep);
+        Sched::SpawnCrashPad(proc->pid);
+
+        // As on x86: a sibling thread's fault must end the whole process.
+        int code = TerminationFromClass(ec);
+        if (Sched::GetCurrentThreadPtr() != proc) {
+            Sched::KillProcess(proc->pid, code);
+        } else {
+            Sched::SetProcessExitCode(proc->pid, code);
+        }
+
+        Sched::ExitProcess();
+        __builtin_unreachable();
+    }
+
+    // A not-present user page: anonymous memory mapped lazily or the user
+    // stack growing. Also reached from the kernel touching user buffers.
+    bool TryResolveUserFault(const Arch::PanicFrame* frame, uint64_t ec) {
+        if (!IsAbort(ec) || !IsTranslationFault(frame->esr)) return false;
+        if (frame->far >= UserSpaceEnd) return false;
+
+        auto* cpu = Smp::TryGetCurrentCpuData();
+        if (cpu == nullptr || cpu->currentSlot < 0 || Sched::GetCurrentPid() < 0) return false;
+
+        bool isData = ec == EcDataAbortLower || ec == EcDataAbortSame;
+        Memory::VMM::PageFault fault{false, isData && (frame->esr & (1ULL << 6)) != 0, !isData};
+        return montauk::abi::TryHandleAnonymousPageFault(frame->far, fault)
+            || Sched::TryGrowUserStack(frame->far);
+    }
+
+}
+
+extern "C" void ArchSyncException(Arch::PanicFrame* frame, bool fromUser) {
+    uint64_t ec = ExceptionClass(frame);
+
+    if (TryResolveUserFault(frame, ec)) {
+        return;
+    }
+
+    if (fromUser && Sched::GetCurrentPid() >= 0) {
+        TerminateUserProcess(frame, ec);
+    }
+
+    Panic(ExceptionName(ec), frame);
+}
+
+// FIQs are never routed, SErrors are asynchronous external aborts, and the
+// kernel neither runs on SP_EL0 nor supports AArch32 user code.
+extern "C" void ArchUnexpectedException(Arch::PanicFrame* frame, uint64_t kind) {
+    static const char* const names[] = {
+        "Exception on SP_EL0", "Unexpected FIQ", "SError (asynchronous abort)",
+        "AArch32 exception",
+    };
+    Panic(kind < 4 ? names[kind] : "Unexpected exception", frame);
+}

kernel/src/Arch/aarch64/Gic.hpp

@@ -0,0 +1,40 @@
+/*
+    * Gic.hpp
+    * aarch64 GICv2 interrupt controller (private to the port)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+#include <ACPI/ACPI.hpp>
+
+namespace Arch::Gic {
+
+    // Bring up the distributor and the calling CPU's interface from the
+    // ACPI MADT. Returns false when no usable GIC is described.
+    bool Initialize(Hal::ACPI::CommonSDTHeader* xsdt);
+
+    bool IsInitialized();
+
+    // The XSDT Initialize was given (for the other ACPI tables the port reads).
+    Hal::ACPI::CommonSDTHeader* CurrentXsdt();
+
+    // Per-CPU part of Initialize, for secondary CPUs.
+    void InitializeCpuInterface();
+
+    // Map a private peripheral interrupt (16-31) onto an IRQ slot and enable
+    // it on the calling CPU.
+    void RoutePpi(uint32_t intid, uint8_t irq, bool edgeTriggered);
+
+    // Called for an IRQ slot after the interrupt is acknowledged and before
+    // it is completed (EOI): level-triggered sources deassert here.
+    using AckHook = void (*)();
+    void SetAckHook(uint8_t irq, AckHook hook);
+
+    // The calling CPU's target bit, as the distributor sees it.
+    uint8_t CurrentTargetMask();
+
+    // Find an ACPI table in the XSDT by signature.
+    Hal::ACPI::CommonSDTHeader* FindTable(Hal::ACPI::CommonSDTHeader* xsdt, const char* signature);
+
+}

kernel/src/Arch/aarch64/InterruptFrame.hpp

@@ -9,7 +9,7 @@
 
 namespace Arch {
 
-    // What the exception vectors save before calling into C++.
+    // What the exception vectors (Vectors.S) save before calling into C++.
     struct PanicFrame {
         uint64_t x[31];   // x0-x30
         uint64_t sp;      // interrupted SP (SP_EL0 for EL0, else SP_EL1)
@@ -19,4 +19,6 @@ namespace Arch {
         uint64_t far;     // FAR_EL1: faulting address
     };
 
+    static_assert(sizeof(PanicFrame) == 288, "must match Vectors.S");
+
 }

kernel/src/Arch/aarch64/Interrupts.cpp

@@ -1,52 +1,420 @@
 /*
     * Interrupts.cpp
-    * aarch64 IRQ slot dispatch (implements Arch/Interrupts.hpp)
+    * aarch64 GICv2 driver and IRQ slot dispatch (implements Arch/Interrupts.hpp)
     * Copyright (c) 2026 Daniel Hammer
 */
 
 #include <Arch/Interrupts.hpp>
+#include <Arch/Smp.hpp>
+#include <Arch/aarch64/Gic.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 
-// IRQ slots are delivered by the interrupt controller driver (GICv2/v3,
-// not written yet), which maps INTIDs onto slots and calls ArchIrqDispatch
-// from the IRQ exception vector after acknowledging the interrupt.
+// IRQ slots map onto GIC interrupt IDs (INTIDs):
+//   IRQ_TIMER            the generic timer's PPI (Timer.cpp routes it)
+//   24-45 (MSI slots)    SPIs of the GICv2m MSI frame, one per slot
+//   IRQ_TLB_SHOOTDOWN,
+//   IRQ_RESCHEDULE       SGIs 0 and 1
+//   1-23 (platform lines) SPIs the PCI layer assigns to INTx pins from
+//                        ACPI _PRT (AssignPlatformLine)
+
+using namespace Kt;
 
 namespace Arch {
 
+    // Distributor registers
+    static constexpr uint32_t GICD_CTLR       = 0x000;
+    static constexpr uint32_t GICD_TYPER      = 0x004;
+    static constexpr uint32_t GICD_ISENABLER  = 0x100;
+    static constexpr uint32_t GICD_ICENABLER  = 0x180;
+    static constexpr uint32_t GICD_ICPENDR    = 0x280;
+    static constexpr uint32_t GICD_ICACTIVER  = 0x380;
+    static constexpr uint32_t GICD_IPRIORITYR = 0x400;
+    static constexpr uint32_t GICD_ITARGETSR  = 0x800;
+    static constexpr uint32_t GICD_ICFGR      = 0xC00;
+    static constexpr uint32_t GICD_SGIR       = 0xF00;
+
+    // CPU interface registers
+    static constexpr uint32_t GICC_CTLR = 0x00;
+    static constexpr uint32_t GICC_PMR  = 0x04;
+    static constexpr uint32_t GICC_BPR  = 0x08;
+    static constexpr uint32_t GICC_IAR  = 0x0C;
+    static constexpr uint32_t GICC_EOIR = 0x10;
+
+    // GICv2m MSI frame registers
+    static constexpr uint32_t V2M_MSI_TYPER     = 0x008;
+    static constexpr uint32_t V2M_MSI_SETSPI_NS = 0x040;
+
+    static constexpr uint32_t SpuriousIntid = 1020;
+    static constexpr uint32_t MaxIntids = 1020;
+    static constexpr uint8_t DefaultPriority = 0xA0;
+    static constexpr uint8_t NoSlot = 0xFF;
+
+    static constexpr uint8_t FirstMsiSlot = 24;
+    static constexpr uint8_t LastMsiSlot = 45;
+
+    // MADT interrupt-controller structures (ACPI 6.x)
+    static constexpr uint8_t MadtGicc = 0x0B;
+    static constexpr uint8_t MadtGicd = 0x0C;
+    static constexpr uint8_t MadtGicMsiFrame = 0x0D;
+
+    struct MadtEntry {
+        uint8_t Type;
+        uint8_t Length;
+    } __attribute__((packed));
+
+    struct MadtGiccEntry {
+        MadtEntry Header;
+        uint16_t Reserved;
+        uint32_t CpuInterfaceNumber;
+        uint32_t AcpiProcessorUid;
+        uint32_t Flags;
+        uint32_t ParkingProtocolVersion;
+        uint32_t PerformanceInterruptGsiv;
+        uint64_t ParkedAddress;
+        uint64_t PhysicalBaseAddress;  // GICC
+        uint64_t Gicv;
+        uint64_t Gich;
+        uint32_t VgicMaintenanceInterrupt;
+        uint64_t GicrBaseAddress;
+        uint64_t Mpidr;
+    } __attribute__((packed));
+
+    struct MadtGicdEntry {
+        MadtEntry Header;
+        uint16_t Reserved;
+        uint32_t GicId;
+        uint64_t PhysicalBaseAddress;
+        uint32_t SystemVectorBase;
+        uint8_t  GicVersion;
+    } __attribute__((packed));
+
+    struct MadtMsiFrameEntry {
+        MadtEntry Header;
+        uint16_t Reserved;
+        uint32_t FrameId;
+        uint64_t PhysicalBaseAddress;
+        uint32_t Flags;                 // bit 0: SpiCount/SpiBase override TYPER
+        uint16_t SpiCount;
+        uint16_t SpiBase;
+    } __attribute__((packed));
+
+    static constexpr uint32_t MadtHeaderSize = 44;  // SDT header + LAPIC address + flags
+
     static IrqHandler g_irqHandlers[IRQ_COUNT] = {};
+    static Gic::AckHook g_ackHooks[IRQ_COUNT] = {};
+    static uint8_t g_intidToSlot[MaxIntids];
+    static uint32_t g_slotToIntid[IRQ_COUNT];
+
+    static volatile uint8_t* g_gicd = nullptr;
+    static volatile uint8_t* g_gicc = nullptr;
+    static uint32_t g_intidCount = 0;
+
+    static Hal::ACPI::CommonSDTHeader* g_xsdt = nullptr;
+
+    struct PlatformLine {
+        uint32_t gsi;       // 0: unassigned
+        bool edgeTriggered;
+    };
+    static PlatformLine g_platformLines[24] = {};
+
+    static uint64_t g_msiFramePhys = 0;
+    static uint32_t g_msiSpiBase = 0;
+    static uint32_t g_msiSpiCount = 0;
+
+    static inline uint32_t GicdRead(uint32_t reg) {
+        return *(volatile uint32_t*)(g_gicd + reg);
+    }
+
+    static inline void GicdWrite(uint32_t reg, uint32_t value) {
+        *(volatile uint32_t*)(g_gicd + reg) = value;
+    }
+
+    static inline void GicdWrite8(uint32_t reg, uint8_t value) {
+        *(volatile uint8_t*)(g_gicd + reg) = value;
+    }
+
+    static inline uint8_t GicdRead8(uint32_t reg) {
+        return *(volatile uint8_t*)(g_gicd + reg);
+    }
+
+    static inline uint32_t GiccRead(uint32_t reg) {
+        return *(volatile uint32_t*)(g_gicc + reg);
+    }
+
+    static inline void GiccWrite(uint32_t reg, uint32_t value) {
+        *(volatile uint32_t*)(g_gicc + reg) = value;
+    }
+
+    static volatile uint8_t* MapRegisters(uint64_t phys, uint64_t size) {
+        for (uint64_t offset = 0; offset < size; offset += 0x1000) {
+            Memory::VMM::g_paging->MapMMIO(phys + offset, Memory::HHDM(phys + offset));
+        }
+        return (volatile uint8_t*)Memory::HHDM(phys);
+    }
+
+    static void SetConfig(uint32_t intid, bool edgeTriggered) {
+        uint32_t reg = GICD_ICFGR + (intid / 16) * 4;
+        uint32_t shift = (intid % 16) * 2 + 1;
+        uint32_t value = GicdRead(reg);
+        if (edgeTriggered) value |= 1u << shift;
+        else value &= ~(1u << shift);
+        GicdWrite(reg, value);
+    }
+
+    static void EnableIntid(uint32_t intid) {
+        GicdWrite(GICD_ISENABLER + (intid / 32) * 4, 1u << (intid % 32));
+    }
+
+    static void Bind(uint32_t intid, uint8_t irq) {
+        if (intid >= MaxIntids || irq >= IRQ_COUNT) return;
+        g_intidToSlot[intid] = irq;
+        g_slotToIntid[irq] = intid;
+    }
+
+    // Route a shared peripheral interrupt to the boot CPU and unmask it.
+    static void RouteSpi(uint32_t intid, uint8_t irq, bool edgeTriggered) {
+        if (intid < 32 || intid >= g_intidCount) return;
+        Bind(intid, irq);
+        GicdWrite8(GICD_IPRIORITYR + intid, DefaultPriority);
+        GicdWrite8(GICD_ITARGETSR + intid, Smp::GetCpuData(0)->arch.gicTargetMask);
+        SetConfig(intid, edgeTriggered);
+        EnableIntid(intid);
+    }
+
+    namespace Gic {
+
+        Hal::ACPI::CommonSDTHeader* FindTable(Hal::ACPI::CommonSDTHeader* xsdt, const char* signature) {
+            if (xsdt == nullptr) return nullptr;
+            uint32_t count = (xsdt->Length - sizeof(Hal::ACPI::CommonSDTHeader)) / 8;
+            auto* entries = (uint64_t*)((uint64_t)xsdt + sizeof(Hal::ACPI::CommonSDTHeader));
+            for (uint32_t i = 0; i < count; i++) {
+                auto* header = (Hal::ACPI::CommonSDTHeader*)Memory::HHDM(entries[i]);
+                if (header->Signature[0] == signature[0] && header->Signature[1] == signature[1]
+                 && header->Signature[2] == signature[2] && header->Signature[3] == signature[3]) {
+                    return header;
+                }
+            }
+            return nullptr;
+        }
+
+        // ITARGETSR0-7 are banked: each byte reads back the reading CPU's
+        // own interface bit.
+        uint8_t CurrentTargetMask() {
+            uint32_t value = GicdRead(GICD_ITARGETSR);
+            return (uint8_t)(value | value >> 8 | value >> 16 | value >> 24);
+        }
+
+        void InitializeCpuInterface() {
+            // SGIs and PPIs are banked per CPU: disable the PPIs until a
+            // driver routes one, keep the SGIs (IPIs) enabled.
+            GicdWrite(GICD_ICENABLER, 0xFFFF0000u);
+            GicdWrite(GICD_ISENABLER, 0x0000FFFFu);
+            GicdWrite(GICD_ICPENDR, 0xFFFFFFFFu);
+            for (uint32_t intid = 0; intid < 32; intid++) {
+                GicdWrite8(GICD_IPRIORITYR + intid, DefaultPriority);
+            }
+
+            GiccWrite(GICC_PMR, 0xF0);
+            GiccWrite(GICC_BPR, 0);
+            GiccWrite(GICC_CTLR, 1);
+
+            auto* cpu = Smp::GetCurrentCpuData();
+            cpu->arch.gicTargetMask = CurrentTargetMask();
+        }
+
+        void RoutePpi(uint32_t intid, uint8_t irq, bool edgeTriggered) {
+            if (intid < 16 || intid >= 32) return;
+            Bind(intid, irq);
+            SetConfig(intid, edgeTriggered);
+            EnableIntid(intid);
+        }
+
+        void SetAckHook(uint8_t irq, AckHook hook) {
+            if (irq < IRQ_COUNT) g_ackHooks[irq] = hook;
+        }
+
+        bool IsInitialized() {
+            return g_gicc != nullptr;
+        }
+
+        Hal::ACPI::CommonSDTHeader* CurrentXsdt() {
+            return g_xsdt;
+        }
+
+        bool Initialize(Hal::ACPI::CommonSDTHeader* xsdt) {
+            g_xsdt = xsdt;
+            auto* madt = FindTable(xsdt, "APIC");
+            if (madt == nullptr) {
+                KernelLogStream(ERROR, "GIC") << "No MADT";
+                return false;
+            }
+
+            uint64_t gicdPhys = 0, giccPhys = 0;
+            uint8_t version = 0;
+            auto* cursor = (uint8_t*)madt + MadtHeaderSize;
+            auto* end = (uint8_t*)madt + madt->Length;
+            while (cursor + sizeof(MadtEntry) <= end) {
+                auto* entry = (MadtEntry*)cursor;
+                if (entry->Length < sizeof(MadtEntry)) break;
+
+                if (entry->Type == MadtGicd) {
+                    auto* gicd = (MadtGicdEntry*)entry;
+                    gicdPhys = gicd->PhysicalBaseAddress;
+                    version = gicd->GicVersion;
+                } else if (entry->Type == MadtGicc && giccPhys == 0) {
+                    giccPhys = ((MadtGiccEntry*)entry)->PhysicalBaseAddress;
+                } else if (entry->Type == MadtGicMsiFrame && g_msiFramePhys == 0) {
+                    auto* frame = (MadtMsiFrameEntry*)entry;
+                    g_msiFramePhys = frame->PhysicalBaseAddress;
+                    if (frame->Flags & 1) {
+                        g_msiSpiBase = frame->SpiBase;
+                        g_msiSpiCount = frame->SpiCount;
+                    }
+                }
+                cursor += entry->Length;
+            }
+
+            // Version 0 means "infer from the hardware"; 1 and 2 share a
+            // programming model.
+            if (gicdPhys == 0 || giccPhys == 0 || version > 2) {
+                KernelLogStream(ERROR, "GIC") << "No GICv2 in the MADT (version "
+                    << base::dec << (uint64_t)version << ")";
+                return false;
+            }
+
+            for (uint32_t i = 0; i < MaxIntids; i++) g_intidToSlot[i] = NoSlot;
+            for (int i = 0; i < IRQ_COUNT; i++) g_slotToIntid[i] = SpuriousIntid;
+            Bind(0, IRQ_TLB_SHOOTDOWN);
+            Bind(1, IRQ_RESCHEDULE);
+
+            g_gicd = MapRegisters(gicdPhys, 0x1000);
+            g_gicc = MapRegisters(giccPhys, 0x2000);
+
+            g_intidCount = ((GicdRead(GICD_TYPER) & 0x1F) + 1) * 32;
+            if (g_intidCount > MaxIntids) g_intidCount = MaxIntids;
+
+            // Distributor: everything masked, idle, level-triggered and
+            // aimed at no CPU until RouteSpi claims it.
+            GicdWrite(GICD_CTLR, 0);
+            for (uint32_t intid = 32; intid < g_intidCount; intid += 32) {
+                GicdWrite(GICD_ICENABLER + (intid / 32) * 4, 0xFFFFFFFFu);
+                GicdWrite(GICD_ICPENDR + (intid / 32) * 4, 0xFFFFFFFFu);
+                GicdWrite(GICD_ICACTIVER + (intid / 32) * 4, 0xFFFFFFFFu);
+            }
+            for (uint32_t intid = 32; intid < g_intidCount; intid++) {
+                GicdWrite8(GICD_IPRIORITYR + intid, DefaultPriority);
+                GicdWrite8(GICD_ITARGETSR + intid, 0);
+            }
+            for (uint32_t intid = 32; intid < g_intidCount; intid += 16) {
+                GicdWrite(GICD_ICFGR + (intid / 16) * 4, 0);
+            }
+            GicdWrite(GICD_CTLR, 1);
+
+            InitializeCpuInterface();
+
+            if (g_msiFramePhys != 0) {
+                volatile uint8_t* frame = MapRegisters(g_msiFramePhys, 0x1000);
+                if (g_msiSpiCount == 0) {
+                    uint32_t typer = *(volatile uint32_t*)(frame + V2M_MSI_TYPER);
+                    g_msiSpiBase = (typer >> 16) & 0x3FF;
+                    g_msiSpiCount = typer & 0x3FF;
+                }
+            }
+
+            KernelLogStream(OK, "GIC") << "GICv2: distributor 0x" << base::hex << gicdPhys
+                << ", CPU interface 0x" << giccPhys << ", " << base::dec
+                << (uint64_t)g_intidCount << " interrupt IDs";
+            if (g_msiSpiCount != 0) {
+                KernelLogStream(OK, "GIC") << "GICv2m MSI frame 0x" << base::hex << g_msiFramePhys
+                    << ": SPIs " << base::dec << (uint64_t)g_msiSpiBase << "-"
+                    << (uint64_t)(g_msiSpiBase + g_msiSpiCount - 1);
+            }
+            return true;
+        }
+
+    }
 
     void RegisterIrqHandler(uint8_t irq, IrqHandler handler) {
         if (irq >= IRQ_COUNT) return;
         g_irqHandlers[irq] = handler;
-        Kt::KernelLogStream(Kt::DEBUG, "IRQ") << "Registered handler for IRQ "
+        KernelLogStream(DEBUG, "IRQ") << "Registered handler for IRQ "
             << base::dec << (uint64_t)irq;
     }
 
-    // TODO(aarch64): GIC SPI routing (ACPI MADT GICD / interrupt overrides).
+    bool PlatformLinesAreFixed() {
+        return false;
+    }
+
+    // ACPI GSIs are GIC INTIDs. The GIC knows only active-high levels and
+    // rising edges; an active-low line is inverted on its way in.
+    bool AssignPlatformLine(uint8_t irq, uint32_t gsi, bool levelTriggered, bool) {
+        if (irq == 0 || irq >= 24 || gsi < 32 || gsi >= g_intidCount) return false;
+        g_platformLines[irq] = PlatformLine{ gsi, !levelTriggered };
+        return true;
+    }
+
     void RouteLegacyIrq(uint8_t irq) {
-        Kt::KernelLogStream(Kt::WARNING, "IRQ") << "aarch64: cannot route IRQ "
-            << base::dec << (uint64_t)irq << " (no GIC driver)";
+        if (irq >= 24 || g_platformLines[irq].gsi == 0) {
+            KernelLogStream(WARNING, "IRQ") << "aarch64: platform line "
+                << base::dec << (uint64_t)irq << " is not assigned";
+            return;
+        }
+        RouteSpi(g_platformLines[irq].gsi, irq, g_platformLines[irq].edgeTriggered);
     }
 
     void EnableLegacyIrq(uint8_t irq) {
         RouteLegacyIrq(irq);
     }
 
-    // TODO(aarch64): GICv3 ITS / GICv2m doorbell.
-    bool ComposeMsi(uint8_t, MsiMessage&) {
-        return false;
+    bool ComposeMsi(uint8_t irq, MsiMessage& out) {
+        if (g_gicd == nullptr || irq < FirstMsiSlot || irq > LastMsiSlot) return false;
+        uint32_t index = irq - FirstMsiSlot;
+        if (index >= g_msiSpiCount) return false;
+
+        uint32_t spi = g_msiSpiBase + index;
+        if (g_slotToIntid[irq] != spi) {
+            RouteSpi(spi, irq, true);
+        }
+        out = MsiMessage{ g_msiFramePhys + V2M_MSI_SETSPI_NS, spi };
+        return true;
     }
 
-    // TODO(aarch64): GIC software-generated interrupts.
-    bool SendIpi(int, uint8_t) {
-        return false;
+    bool SendIpi(int cpuIndex, uint8_t irq) {
+        if (g_gicd == nullptr) return false;
+        auto* cpu = Smp::GetCpuData(cpuIndex);
+        if (cpu == nullptr || cpu->arch.gicTargetMask == 0) return false;
+        uint32_t sgi = g_slotToIntid[irq];
+        if (sgi >= 16) return false;
+
+        // Make the IPI's memory effects visible before it is raised.
+        asm volatile("dsb ishst" ::: "memory");
+        GicdWrite(GICD_SGIR, ((uint32_t)cpu->arch.gicTargetMask << 16) | sgi);
+        return true;
     }
 
 }
 
-extern "C" void ArchIrqDispatch(uint64_t irq, bool fromUser) {
-    if (irq < Arch::IRQ_COUNT && Arch::g_irqHandlers[irq] != nullptr) {
-        Arch::g_irqHandlers[irq]((uint8_t)irq, fromUser);
+// IRQ exception vector (Vectors.S). As on x86, the interrupt is completed
+// BEFORE the handler runs: the handler may context-switch away and not
+// come back here for a long time, and an un-EOId interrupt would hold this
+// CPU's running priority up (no further ticks). Level-triggered sources
+// get the chance to drop their line first through an ack hook.
+extern "C" void ArchIrqException(void*, bool fromUser) {
+    using namespace Arch;
+    uint32_t iar = GiccRead(GICC_IAR);
+    uint32_t intid = iar & 0x3FF;
+    if (intid >= SpuriousIntid) return;
+
+    uint8_t irq = g_intidToSlot[intid];
+    if (irq != NoSlot && g_ackHooks[irq] != nullptr) {
+        g_ackHooks[irq]();
+    }
+    GiccWrite(GICC_EOIR, iar);
+
+    if (irq != NoSlot && g_irqHandlers[irq] != nullptr) {
+        g_irqHandlers[irq](irq, fromUser);
     }
 }

kernel/src/Arch/aarch64/Paging.cpp

@@ -5,29 +5,68 @@
 */
 
 #include <Memory/Paging.hpp>
+#include <Memory/PageFrameAllocator.hpp>
+#include <Memory/HHDM.hpp>
 #include <Arch/Cpu.hpp>
 #include <Common/Panic.hpp>
 #include <CppLib/Spinlock.hpp>
+#include <Arch/aarch64/Bringup.hpp>
 
-// TODO(aarch64): 4 KiB-granule, 48-bit VA translation tables. The kernel
-// half lives in TTBR1_EL1 and is never switched; "address space roots"
-// handed to the generic kernel are TTBR0_EL1 tables, so
-// CreateUserAddressSpace has no kernel half to copy. MAIR_EL1 needs
-// Normal-WB, Normal-NC (MapWC) and Device-nGnRE (MapMMIO) indices.
+// 4 KiB granule, 48-bit virtual addresses, four levels (0-3). The kernel
+// half is translated through TTBR1_EL1 (g_kernelHalfRoot), which never
+// changes after Init. The "address space roots" the generic kernel switches
+// between are TTBR0_EL1 tables: a process's root holds only its own lower
+// half, and KernelRoot() is the TTBR0 table of kernel contexts, empty apart
+// from the EFI runtime identity mappings (so null pointers fault).
+//
+// The HHDM covers only what the memory map lists as RAM or firmware data:
+// unlike on x86, the holes between are device space on ARM machines, and a
+// cacheable mapping of device registers invites speculative accesses.
 
 namespace Memory::VMM {
 
     Paging* g_paging = nullptr;
 
+    static std::uint64_t g_hhdmLimit = 0;
+
+    static std::uint64_t g_kernelHalfRoot = 0;
+
+    std::uint64_t Paging::HhdmLimit() { return g_hhdmLimit; }
+
+    // Translation table descriptor bits (VMSAv8-64, stage 1)
+    static constexpr uint64_t DescValid      = 1ULL << 0;
+    static constexpr uint64_t DescTable      = 1ULL << 1;   // table (levels 0-2) or page (level 3)
+    static constexpr uint64_t DescAttrShift  = 2;           // AttrIndx[2:0], into MAIR_EL1
+    static constexpr uint64_t DescAttrMask   = 7ULL << DescAttrShift;
+    static constexpr uint64_t DescApUser     = 1ULL << 6;   // AP[1]: EL0 may access
+    static constexpr uint64_t DescApReadOnly = 1ULL << 7;   // AP[2]
+    static constexpr uint64_t DescShInner    = 3ULL << 8;
+    static constexpr uint64_t DescAf         = 1ULL << 10;  // access flag (no AF faults)
+    static constexpr uint64_t DescNotGlobal  = 1ULL << 11;
+    static constexpr uint64_t DescPxn        = 1ULL << 53;
+    static constexpr uint64_t DescUxn        = 1ULL << 54;
+    static constexpr uint64_t DescAddrMask   = 0x0000FFFFFFFFF000ULL;
+
+    // MAIR_EL1 attribute indices
+    static constexpr uint64_t AttrNormal   = 0;  // write-back, read/write-allocate
+    static constexpr uint64_t AttrNormalNc = 1;  // non-cacheable (write-combining)
+    static constexpr uint64_t AttrDevice   = 2;  // Device-nGnRE
+    static constexpr uint64_t MairValue = 0xFFULL << (8 * AttrNormal)
+                                        | 0x44ULL << (8 * AttrNormalNc)
+                                        | 0x04ULL << (8 * AttrDevice);
+
+    static constexpr uint64_t KernelHalfBase = 0xFFFF000000000000ULL;
+    static constexpr uint64_t UserSpaceEnd   = 0x0000800000000000ULL;
+
+    // EFI memory types whose runtime regions are device registers
+    static constexpr uint32_t EfiMemoryMappedIo          = 11;
+    static constexpr uint32_t EfiMemoryMappedIoPortSpace = 12;
+
     static kcp::Mutex pagingLock;
 
     void LockUserPaging()   { pagingLock.Acquire(); }
     void UnlockUserPaging() { pagingLock.Release(); }
 
-    [[noreturn]] static void NotImplemented(const char* what) {
-        Panic(what, nullptr);
-    }
-
     void ActivateKernelAddressSpace() {
         Arch::WritePageTableRoot(g_paging->KernelRoot());
     }
@@ -36,75 +75,454 @@ namespace Memory::VMM {
         Arch::FlushTlbAll();
     }
 
+    using Descriptor = uint64_t;
+
+    static inline volatile Descriptor* TableAt(uint64_t phys) {
+        return (volatile Descriptor*)Memory::HHDM(phys);
+    }
+
+    static inline uint64_t Index(uint64_t va, int level) {
+        return (va >> (39 - 9 * level)) & 0x1FF;
+    }
+
+    // Order descriptor writes before the table walker (and later
+    // instructions) can observe them.
+    static inline void PublishTables() {
+        asm volatile("dsb ishst\n\tisb" ::: "memory");
+    }
+
+    static inline uint64_t ReadTtbr1() {
+        uint64_t ttbr;
+        asm volatile("mrs %0, ttbr1_el1" : "=r"(ttbr));
+        return ttbr & Arch::TtbrAddressMask;
+    }
+
+    // The level-3 descriptor for `va`, creating intermediate tables.
+    // Returns nullptr when out of memory or when a block mapping covers
+    // `va` (only the bootloader's tables contain blocks).
+    static volatile Descriptor* WalkCreate(uint64_t root, uint64_t va) {
+        uint64_t table = root;
+        for (int level = 0; level < 3; level++) {
+            volatile Descriptor* entry = &TableAt(table)[Index(va, level)];
+            Descriptor desc = *entry;
+            if ((desc & DescValid) == 0) {
+                void* page = Memory::g_pfa->AllocateZeroed();
+                if (page == nullptr) return nullptr;
+                table = Memory::SubHHDM(page);
+                // The zeroed table must be visible before the walker can
+                // reach it.
+                asm volatile("dsb ishst" ::: "memory");
+                *entry = table | DescTable | DescValid;
+            } else if ((desc & DescTable) == 0) {
+                return nullptr;
+            } else {
+                table = desc & DescAddrMask;
+            }
+        }
+        return &TableAt(table)[Index(va, 3)];
+    }
+
+    // The level-3 descriptor for `va`, or nullptr if a table is missing.
+    static volatile Descriptor* WalkExisting(uint64_t root, uint64_t va) {
+        uint64_t table = root;
+        for (int level = 0; level < 3; level++) {
+            Descriptor desc = TableAt(table)[Index(va, level)];
+            if ((desc & (DescValid | DescTable)) != (DescValid | DescTable)) return nullptr;
+            table = desc & DescAddrMask;
+        }
+        return &TableAt(table)[Index(va, 3)];
+    }
+
+    // Install a leaf. Replacing a live translation (other address or
+    // attributes) needs break-before-make: invalidate, flush, then write.
+    static void SetLeaf(volatile Descriptor* entry, Descriptor desc, uint64_t va) {
+        Descriptor old = *entry;
+        if (old == desc) return;
+        if (old & DescValid) {
+            *entry = 0;
+            Arch::FlushTlbPage(va);
+        }
+        *entry = desc;
+        PublishTables();
+    }
+
+    static Descriptor KernelLeaf(uint64_t phys, uint64_t attr) {
+        Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
+                        | (attr << DescAttrShift) | DescUxn;
+        // Kernel memory stays executable, as on x86; device memory never is.
+        if (attr != AttrNormal) desc |= DescPxn;
+        return desc;
+    }
+
+    static Descriptor UserLeaf(uint64_t phys, uint64_t attr, bool writable, bool executable) {
+        Descriptor desc = (phys & DescAddrMask) | DescTable | DescValid | DescAf | DescShInner
+                        | DescNotGlobal | (attr << DescAttrShift) | DescApUser | DescPxn;
+        if (!writable) desc |= DescApReadOnly;
+        if (!executable) desc |= DescUxn;
+        return desc;
+    }
+
+    static inline bool IsUserLeaf(Descriptor desc) {
+        return (desc & (DescValid | DescTable)) == (DescValid | DescTable) && (desc & DescApUser);
+    }
+
+    // Translate through a table tree that may contain block mappings (the
+    // bootloader's). Returns 0 if unmapped.
+    static uint64_t Translate(uint64_t root, uint64_t va) {
+        uint64_t table = root;
+        for (int level = 0; level <= 3; level++) {
+            Descriptor desc = TableAt(table)[Index(va, level)];
+            if ((desc & DescValid) == 0) return 0;
+
+            uint64_t span = 1ULL << (39 - 9 * level);
+            bool leaf = level == 3 || (desc & DescTable) == 0;
+            if (leaf) {
+                if (level == 0) return 0;  // no level-0 blocks with 4 KiB granules
+                return ((desc & DescAddrMask) & ~(span - 1)) | (va & (span - 1));
+            }
+            table = desc & DescAddrMask;
+        }
+        return 0;
+    }
+
+    void Paging::MapEfiRuntime(const montauk::boot::EfiInfo& efi) {
+        if (efi.memoryMap == nullptr || efi.descriptorSize == 0) return;
+
+        struct EfiMemDesc {
+            uint32_t Type;
+            uint64_t PhysicalStart;
+            uint64_t VirtualStart;
+            uint64_t NumberOfPages;
+            uint64_t Attribute;
+        };
+
+        static constexpr uint64_t EFI_MEMORY_RUNTIME = 0x8000000000000000ULL;
+
+        auto* base = (uint8_t*)efi.memoryMap;
+        uint64_t count = efi.memoryMapSize / efi.descriptorSize;
+
+        for (uint64_t i = 0; i < count; i++) {
+            auto* desc = (EfiMemDesc*)(base + i * efi.descriptorSize);
+            if ((desc->Attribute & EFI_MEMORY_RUNTIME) == 0) continue;
+
+            bool device = desc->Type == EfiMemoryMappedIo || desc->Type == EfiMemoryMappedIoPortSpace;
+            for (uint64_t p = 0; p < desc->NumberOfPages; p++) {
+                uint64_t addr = desc->PhysicalStart + p * 0x1000;
+                // Firmware code runs at its HHDM alias (Efi::Init calls the
+                // services there) and reaches its data at physical addresses.
+                // Runtime regions are reserved memory, outside the HHDM.
+                if (device) {
+                    MapMMIO(addr, addr);
+                    MapMMIO(addr, Memory::HHDM(addr));
+                } else {
+                    Map(addr, addr);
+                    Map(addr, Memory::HHDM(addr));
+                }
+            }
+        }
+
+        FlushTLB();
+    }
+
+}
+
+namespace Arch {
+
+    // The new tables map the code running here at the same address as the
+    // bootloader's did, so the switch is a register write followed by a
+    // full invalidate.
+    void ActivateKernelTranslation() {
+        using namespace Memory::VMM;
+
+        // 48-bit VAs in both halves, 4 KiB granules, write-back walks of
+        // inner-shareable tables; the physical size is what the CPU has.
+        uint64_t mmfr0;
+        asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0));
+        uint64_t ips = mmfr0 & 0xF;
+        if (ips > 5) ips = 5;  // 52-bit output needs LPA2 descriptors
+        uint64_t tcr = 16ULL                          // T0SZ
+                     | 1ULL << 8 | 1ULL << 10        // IRGN0, ORGN0: write-back
+                     | 3ULL << 12                    // SH0: inner shareable
+                     | 0ULL << 14                    // TG0: 4 KiB
+                     | 16ULL << 16                   // T1SZ
+                     | 1ULL << 24 | 1ULL << 26       // IRGN1, ORGN1
+                     | 3ULL << 28                    // SH1
+                     | 2ULL << 30                    // TG1: 4 KiB
+                     | ips << 32;                    // IPS
+
+        asm volatile(
+            "dsb ish\n\t"
+            "msr mair_el1, %0\n\t"
+            "msr tcr_el1, %1\n\t"
+            "isb\n\t"
+            "msr ttbr1_el1, %2\n\t"
+            "msr ttbr0_el1, %3\n\t"
+            "isb\n\t"
+            "tlbi vmalle1\n\t"
+            "dsb nsh\n\t"
+            "isb"
+            :: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(g_paging->KernelRoot()) : "memory");
+    }
+
+}
+
+namespace Memory::VMM {
+
     Paging::Paging() {
-        NotImplemented("aarch64: virtual memory not implemented");
+        rootPhys = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
+        g_kernelHalfRoot = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
     }
 
-    void Paging::Init(std::uint64_t, std::uint64_t, const montauk::boot::MemoryMap&,
-                      const montauk::boot::Framebuffer&) {
-        NotImplemented("aarch64: Paging::Init not implemented");
+    // Kernel mappings of lower-half addresses (EFI identity mappings) go to
+    // the kernel contexts' TTBR0 table.
+    static uint64_t KernelRootFor(uint64_t kernelContextRoot, uint64_t va) {
+        return va >= KernelHalfBase ? g_kernelHalfRoot : kernelContextRoot;
     }
 
-    std::uint64_t Paging::HhdmLimit() { return 0; }
-
-    void Paging::Map(std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::Map not implemented");
+    void Paging::Init(std::uint64_t kernelBaseVirt, std::uint64_t kernelSize, const montauk::boot::MemoryMap& memMap,
+                      const montauk::boot::Framebuffer& framebuffer) {
+        if (Memory::HHDMBase < KernelHalfBase) {
+            Panic("aarch64: HHDM outside the 48-bit kernel half", nullptr);
         }
 
-    void Paging::MapMMIO(std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::MapMMIO not implemented");
+        uint64_t bootTtbr1 = ReadTtbr1();
+        uint64_t bootTcr, bootMair;
+        asm volatile("mrs %0, tcr_el1" : "=r"(bootTcr));
+        asm volatile("mrs %0, mair_el1" : "=r"(bootMair));
+        Kt::KernelLogStream(Kt::DEBUG, "VMM") << "Bootloader TCR 0x" << base::hex << bootTcr
+            << ", MAIR 0x" << bootMair << ", TTBR1 0x" << bootTtbr1;
+
+        // The kernel image, at the physical pages the bootloader loaded it to.
+        for (uint64_t va = kernelBaseVirt; va < kernelBaseVirt + kernelSize; va += 0x1000) {
+            uint64_t phys = Translate(bootTtbr1, va);
+            if (phys == 0) Panic("aarch64: kernel page missing from the bootloader's tables", nullptr);
+            Map(phys, va);
         }
 
-    void Paging::MapWC(std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::MapWC not implemented");
+        // HHDM: every region that is real backing store (see the top of
+        // this file). The limit is published for the AML interpreter.
+        uint64_t maxPhysAddr = 0;
+        for (size_t i = 0; i < memMap.count; i++) {
+            const auto& region = memMap.regions[i];
+            if (region.kind == montauk::boot::MemoryKind::Reserved
+             || region.kind == montauk::boot::MemoryKind::Unknown) {
+                continue;
             }
 
-    void Paging::MapUser(std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::MapUser not implemented");
+            uint64_t start = region.base & ~0xFFFULL;
+            uint64_t end = (region.base + region.length + 0xFFF) & ~0xFFFULL;
+            bool framebufferRegion = region.kind == montauk::boot::MemoryKind::Framebuffer;
+            for (uint64_t phys = start; phys < end; phys += 0x1000) {
+                if (framebufferRegion) MapWC(phys, Memory::HHDM(phys));
+                else Map(phys, Memory::HHDM(phys));
+            }
+            if (end > maxPhysAddr) maxPhysAddr = end;
+        }
+        g_hhdmLimit = maxPhysAddr;
+
+        // The framebuffer may not be listed in the memory map at all.
+        if (framebuffer.address != 0) {
+            uint64_t fbPhys = Memory::SubHHDM(framebuffer.address) & ~0xFFFULL;
+            uint64_t fbPages = (framebuffer.height * framebuffer.pitch + 0xFFF) / 0x1000;
+            for (uint64_t p = 0; p < fbPages; p++) {
+                uint64_t phys = fbPhys + p * 0x1000;
+                MapWC(phys, Memory::HHDM(phys));
+            }
         }
 
-    std::uint64_t Paging::GetPhysAddr(std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::GetPhysAddr not implemented");
+        // The bootloader wrote the image through the data cache. Push it to
+        // the point of unification and drop every instruction-cache line
+        // before the kernel relies on fetching code it has not run yet: a
+        // line still dirty in L1D is invisible to instruction fetch, which
+        // then reads stale DRAM (an "unknown reason" exception on a valid
+        // instruction).
+        Arch::SyncInstructionCache((const void*)kernelBaseVirt, kernelSize);
+
+        Arch::ActivateKernelTranslation();
+
+        Kt::KernelLogStream(Kt::OK, "VMM") << "Switched to kernel translation tables (HHDM limit 0x"
+            << base::hex << g_hhdmLimit << ")";
+    }
+
+    static void MapKernelPage(uint64_t root, uint64_t phys, uint64_t va, uint64_t attr, const char* what) {
+        if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
+            Panic(what, nullptr);
+        }
+        volatile Descriptor* entry = WalkCreate(KernelRootFor(root, va), va);
+        if (entry == nullptr) {
+            Panic("OOM in Paging (kernel page table allocation)", nullptr);
+        }
+        SetLeaf(entry, KernelLeaf(phys, attr), va);
+    }
+
+    void Paging::Map(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormal,
+                      "Value that isn't page-aligned passed as address to Paging::Map!");
+    }
+
+    void Paging::MapMMIO(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrDevice,
+                      "Value that isn't page-aligned passed as address to Paging::MapMMIO!");
+    }
+
+    void Paging::MapWC(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        MapKernelPage(rootPhys, physicalAddress, virtualAddress, AttrNormalNc,
+                      "Value that isn't page-aligned passed as address to Paging::MapWC!");
+    }
+
+    void Paging::MapUser(std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        if (virtualAddress % 0x1000 != 0 || physicalAddress % 0x1000 != 0) {
+            Panic("Value that isn't page-aligned passed as address to Paging::MapUser!", nullptr);
+        }
+        volatile Descriptor* entry = WalkCreate(rootPhys, virtualAddress);
+        if (entry == nullptr) {
+            Panic("OOM in Paging::MapUser (page table allocation)", nullptr);
+        }
+        SetLeaf(entry, UserLeaf(physicalAddress, AttrNormal, true, true), virtualAddress);
+    }
+
+    std::uint64_t Paging::GetPhysAddr(std::uint64_t root, std::uint64_t virtualAddress) {
+        if (virtualAddress >= KernelHalfBase) {
+            root = g_kernelHalfRoot;
+        }
+        return Translate(root, virtualAddress);
     }
 
     std::uint64_t Paging::GetPhysAddr(std::uint64_t virtualAddress) {
         return GetPhysAddr(rootPhys, virtualAddress);
     }
 
+    // The kernel half lives in TTBR1, so a new address space starts empty.
     std::uint64_t Paging::CreateUserAddressSpace() {
-        NotImplemented("aarch64: Paging::CreateUserAddressSpace not implemented");
+        void* page = Memory::g_pfa->AllocateZeroed();
+        if (page == nullptr) return 0;
+        return Memory::SubHHDM(page);
     }
 
-    bool Paging::MapUserIn(std::uint64_t, std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::MapUserIn not implemented");
+    bool Paging::MapUserIn(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        return MapUserInPermissions(root, physicalAddress, virtualAddress, true, true);
     }
 
-    bool Paging::MapUserInPermissions(std::uint64_t, std::uint64_t, std::uint64_t, bool, bool) {
-        NotImplemented("aarch64: Paging::MapUserInPermissions not implemented");
+    static bool MapUserLeaf(uint64_t root, uint64_t phys, uint64_t va, Descriptor desc, const char* what) {
+        pagingLock.Acquire();
+        if (va % 0x1000 != 0 || phys % 0x1000 != 0) {
+            pagingLock.Release();
+            Panic(what, nullptr);
+        }
+        volatile Descriptor* entry = WalkCreate(root, va);
+        if (entry == nullptr) {
+            pagingLock.Release();
+            return false;
+        }
+        SetLeaf(entry, desc, va);
+        pagingLock.Release();
+        return true;
     }
 
-    bool Paging::ProtectUserIn(std::uint64_t, std::uint64_t, bool, bool) {
-        NotImplemented("aarch64: Paging::ProtectUserIn not implemented");
+    bool Paging::MapUserInPermissions(std::uint64_t root, std::uint64_t physicalAddress,
+                                      std::uint64_t virtualAddress, bool writable, bool executable) {
+        return MapUserLeaf(root, physicalAddress, virtualAddress,
+                           UserLeaf(physicalAddress, AttrNormal, writable, executable),
+                           "Non-aligned address in Paging::MapUserIn!");
     }
 
-    bool Paging::MapUserInWC(std::uint64_t, std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::MapUserInWC not implemented");
+    bool Paging::MapUserInWC(std::uint64_t root, std::uint64_t physicalAddress, std::uint64_t virtualAddress) {
+        return MapUserLeaf(root, physicalAddress, virtualAddress,
+                           UserLeaf(physicalAddress, AttrNormalNc, true, false),
+                           "Non-aligned address in Paging::MapUserInWC!");
     }
 
-    void Paging::UnmapUserIn(std::uint64_t, std::uint64_t) {
-        NotImplemented("aarch64: Paging::UnmapUserIn not implemented");
+    // Permission changes need no break-before-make, only the TLB flush.
+    bool Paging::ProtectUserIn(std::uint64_t root, std::uint64_t virtualAddress,
+                               bool writable, bool executable) {
+        if ((virtualAddress & 0xFFFULL) != 0) return false;
+        pagingLock.Acquire();
+        volatile Descriptor* entry = WalkExisting(root, virtualAddress);
+        if (entry == nullptr || !IsUserLeaf(*entry)) {
+            pagingLock.Release();
+            return false;
         }
 
-    void Paging::FreeUserHalf(std::uint64_t) {
-        NotImplemented("aarch64: Paging::FreeUserHalf not implemented");
+        Descriptor desc = *entry;
+        if (writable) desc &= ~DescApReadOnly;
+        else desc |= DescApReadOnly;
+        if (executable) desc &= ~DescUxn;
+        else desc |= DescUxn;
+        *entry = desc;
+        Arch::FlushTlbPage(virtualAddress);
+        pagingLock.Release();
+        return true;
     }
 
-    bool Paging::IsUserRangeAccessible(std::uint64_t, std::uint64_t, std::uint64_t, bool) {
-        NotImplemented("aarch64: Paging::IsUserRangeAccessible not implemented");
+    void Paging::UnmapUserIn(std::uint64_t root, std::uint64_t virtualAddress) {
+        pagingLock.Acquire();
+        volatile Descriptor* entry = WalkExisting(root, virtualAddress);
+        if (entry != nullptr && (*entry & DescValid)) {
+            *entry = 0;
+            Arch::FlushTlbPage(virtualAddress);
+        }
+        pagingLock.Release();
     }
 
-    void Paging::MapEfiRuntime(const montauk::boot::EfiInfo&) {
-        NotImplemented("aarch64: Paging::MapEfiRuntime not implemented");
+    void Paging::FreeUserHalf(std::uint64_t root) {
+        // User space is the lower 128 TiB: level-0 entries 0-255.
+        for (int i0 = 0; i0 < 256; i0++) {
+            Descriptor d0 = TableAt(root)[i0];
+            if ((d0 & DescValid) == 0) continue;
+            uint64_t l1 = d0 & DescAddrMask;
+
+            for (int i1 = 0; i1 < 512; i1++) {
+                Descriptor d1 = TableAt(l1)[i1];
+                if ((d1 & DescValid) == 0) continue;
+                uint64_t l2 = d1 & DescAddrMask;
+
+                for (int i2 = 0; i2 < 512; i2++) {
+                    Descriptor d2 = TableAt(l2)[i2];
+                    if ((d2 & DescValid) == 0) continue;
+                    uint64_t l3 = d2 & DescAddrMask;
+
+                    for (int i3 = 0; i3 < 512; i3++) {
+                        Descriptor leaf = TableAt(l3)[i3];
+                        if ((leaf & DescValid) == 0) continue;
+
+                        // Write-combining/device pages are not the process's.
+                        if (((leaf & DescAttrMask) >> DescAttrShift) != AttrNormal) continue;
+
+                        uint64_t pagePhys = leaf & DescAddrMask;
+                        if (pagePhys != 0) {
+                            Memory::g_pfa->Free((void*)Memory::HHDM(pagePhys));
+                        }
+                    }
+
+                    Memory::g_pfa->Free((void*)Memory::HHDM(l3));
+                }
+
+                Memory::g_pfa->Free((void*)Memory::HHDM(l2));
+            }
+
+            Memory::g_pfa->Free((void*)Memory::HHDM(l1));
+        }
+    }
+
+    bool Paging::IsUserRangeAccessible(std::uint64_t root, std::uint64_t virtualAddress,
+                                       std::uint64_t size, bool requireWrite) {
+        if (size == 0) return true;
+        if (virtualAddress >= UserSpaceEnd) return false;
+        if (virtualAddress + size < virtualAddress) return false;
+        if (virtualAddress + size > UserSpaceEnd) return false;
+
+        uint64_t page = virtualAddress & ~0xFFFULL;
+        uint64_t lastPage = (virtualAddress + size - 1) & ~0xFFFULL;
+        for (;;) {
+            volatile Descriptor* entry = WalkExisting(root, page);
+            if (entry == nullptr) return false;
+            Descriptor desc = *entry;
+            if (!IsUserLeaf(desc)) return false;
+            if (requireWrite && (desc & DescApReadOnly)) return false;
+            if (page == lastPage) break;
+            page += 0x1000;
+        }
+        return true;
     }
 
 }

kernel/src/Arch/aarch64/PanicFrame.cpp

@@ -24,5 +24,13 @@ namespace Arch {
         PrintBoxedHex(kerr, "PSTATE (SPSR)",         frame->spsr,      BoxWidth);
         PrintBoxedHex(kerr, "Stack Pointer",         frame->sp,        BoxWidth);
         PrintBoxedHex(kerr, "Link Register (x30)",   frame->x[30],     BoxWidth);
+
+        // An "unknown reason" exception on a kernel PC: show the word the
+        // data side sees there. A valid instruction means instruction fetch
+        // saw something else (cache maintenance); garbage means the code
+        // itself was overwritten. The fetch succeeded, so the page is mapped.
+        if ((frame->esr >> 26) == 0 && frame->elr >= 0xFFFF000000000000ULL && (frame->elr & 3) == 0) {
+            PrintBoxedHex(kerr, "Instruction at PC",     *(volatile uint32_t*)frame->elr, BoxWidth);
+        }
     }
 }

kernel/src/Arch/aarch64/PerCpu.hpp

@@ -17,6 +17,8 @@ namespace Arch {
 
     struct CpuState {
         uint64_t mpidr;            // MPIDR_EL1 affinity of this CPU
+        uint8_t gicTargetMask;     // GICv2 CPU-interface bit (SGI/SPI targets)
+        uint64_t timerPeriod;      // tick period in counter ticks, 0 = stopped
     };
 
     // TPIDR_EL1 holds the CpuData pointer (zeroed in EarlyInitialize until

kernel/src/Arch/aarch64/Platform.cpp

@@ -7,16 +7,73 @@
 #include <Arch/Platform.hpp>
 #include <Arch/Cpu.hpp>
 #include <Arch/Smp.hpp>
+#include <Arch/aarch64/Gic.hpp>
+#include <Arch/aarch64/Bringup.hpp>
+#include <ACPI/FADT.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 
 using namespace Kt;
 
+extern "C" char ArchExceptionVectors[];  // Vectors.S
+
 namespace Arch {
 
-    void EarlyInitialize() {
-        // No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
-        asm volatile("msr tpidr_el1, xzr");
+    // SCTLR_EL1.WXN: writable memory is execute-never
+    static constexpr uint64_t SctlrWxn = 1ULL << 19;
+    // SCTLR_EL1.UCT/UCI: EL0 may read CTR_EL0 and clean/invalidate caches
+    // by address (what __builtin___clear_cache needs for generated code).
+    static constexpr uint64_t SctlrUct = 1ULL << 15;
+    static constexpr uint64_t SctlrUci = 1ULL << 26;
+
+    // PSCI 0.2+ function IDs (SMC32 calling convention)
+    static constexpr uint64_t PsciSystemOff   = 0x84000008;
+    static constexpr uint64_t PsciSystemReset = 0x84000009;
+
+    // How the firmware takes PSCI calls, from the FADT's ARM_BOOT_ARCH.
+    enum class PsciConduit { None, Smc, Hvc };
+    static PsciConduit g_psci = PsciConduit::None;
+
+    static uint64_t PsciCall(uint64_t function) {
+        register uint64_t x0 asm("x0") = function;
+        register uint64_t x1 asm("x1") = 0;
+        register uint64_t x2 asm("x2") = 0;
+        register uint64_t x3 asm("x3") = 0;
+        if (g_psci == PsciConduit::Hvc) {
+            asm volatile("hvc #0" : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) :: "memory");
+        } else if (g_psci == PsciConduit::Smc) {
+            asm volatile("smc #0" : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) :: "memory");
+        }
+        return x0;
+    }
+
+    static void InitializePsci(Hal::ACPI::CommonSDTHeader* xsdt) {
+        Hal::FADT::ParsedFADT fadt{};
+        if (!Hal::FADT::Parse(xsdt, fadt) || !fadt.Valid) return;
+        if (!(fadt.ArmBootArchFlags & Hal::FADT::ArmBootPsciCompliant)) {
+            KernelLogStream(WARNING, "PSCI") << "Firmware is not PSCI compliant: no reset or power-off";
+            return;
+        }
+        g_psci = (fadt.ArmBootArchFlags & Hal::FADT::ArmBootPsciUseHvc) ? PsciConduit::Hvc : PsciConduit::Smc;
+        KernelLogStream(OK, "PSCI") << "PSCI through " << (g_psci == PsciConduit::Hvc ? "HVC" : "SMC");
+    }
+
+    void InitializeCpuRegisters() {
+        // Run on SP_EL1 so exceptions taken from the kernel arrive on the
+        // stack it is already using (Vectors.S has no SP_EL0 handlers).
+        asm volatile("mov x9, sp\n\tmsr spsel, #1\n\tmov sp, x9" ::: "x9", "memory");
+
+        // The vectors need no allocation, so take over from the
+        // bootloader's right away.
+        asm volatile("msr vbar_el1, %0\n\tisb" :: "r"(ArchExceptionVectors) : "memory");
+
+        // Kernel mappings are RWX (Paging.cpp), so writable must not
+        // imply execute-never; user code may maintain its own caches.
+        uint64_t sctlr;
+        asm volatile("mrs %0, sctlr_el1" : "=r"(sctlr));
+        sctlr &= ~SctlrWxn;
+        sctlr |= SctlrUct | SctlrUci;
+        asm volatile("msr sctlr_el1, %0\n\tisb" :: "r"(sctlr));
 
         // CPACR_EL1.FPEN = 0b11: FP/SIMD usable at EL0 and EL1 (the kernel
         // itself is built -mgeneral-regs-only; Context.S saves user state).
@@ -26,10 +83,17 @@ namespace Arch {
         asm volatile("msr cpacr_el1, %0\n\tisb" :: "r"(cpacr));
     }
 
-    // TODO(aarch64): install the VBAR_EL1 vector table (EL1/EL0 synchronous
-    // exceptions -> panic / SyscallDispatch, IRQ -> GIC -> ArchIrqDispatch).
+    void EarlyInitialize() {
+        // No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
+        asm volatile("msr tpidr_el1, xzr");
+
+        InitializeCpuRegisters();
+    }
+
+    // Installed by EarlyInitialize.
     void InitializeTraps() {
-        KernelLogStream(WARNING, "Arch") << "aarch64: exception vectors not implemented";
+        KernelLogStream(OK, "Arch") << "Installed exception vectors at 0x" << base::hex
+            << (uint64_t)ArchExceptionVectors;
     }
 
     // MAIR_EL1 attribute indices are chosen together with the page-table
@@ -37,10 +101,12 @@ namespace Arch {
     void InitializeMemoryAttributes() {
     }
 
-    // TODO(aarch64): GIC distributor/redistributor from the ACPI MADT.
-    void InitializeInterrupts(Hal::ACPI::CommonSDTHeader*) {
+    void InitializeInterrupts(Hal::ACPI::CommonSDTHeader* xsdt) {
         Smp::InitBsp();
-        KernelLogStream(WARNING, "Arch") << "aarch64: no interrupt controller driver";
+        if (!Gic::Initialize(xsdt)) {
+            KernelLogStream(ERROR, "Arch") << "aarch64: no interrupt controller, running without IRQs";
+        }
+        InitializePsci(xsdt);
     }
 
     void InitializePlatformDevices() {
@@ -53,12 +119,22 @@ namespace Arch {
     void LateInitialize() {
     }
 
-    void ListPlatformDevices(PlatformDeviceSink, void*) {
+    void ListPlatformDevices(PlatformDeviceSink sink, void* context) {
+        if (Gic::IsInitialized()) {
+            sink(context, 1, "GICv2", "ARM Generic Interrupt Controller");
+            sink(context, 2, "Generic Timer", "ARM architected timer (EL1 virtual)");
+        }
     }
 
-    // TODO(aarch64): PSCI SYSTEM_RESET through the conduit the FADT names.
     void Reboot() {
+        PsciCall(PsciSystemReset);
         HaltForever();
     }
 
+    void PowerOff() {
+        if (g_psci != PsciConduit::None) {
+            PsciCall(PsciSystemOff);
+        }
+    }
+
 }

kernel/src/Arch/aarch64/Smp.cpp

@@ -1,13 +1,19 @@
 /*
     * Smp.cpp
-    * aarch64 per-CPU data (implements the arch half of Arch/Smp.hpp)
+    * aarch64 per-CPU data and secondary CPU start-up (implements the arch half of Arch/Smp.hpp)
     * Copyright (c) 2026 Daniel Hammer
 */
 
 #include <Arch/Smp.hpp>
+#include <Arch/Cpu.hpp>
+#include <Arch/Syscall.hpp>
+#include <Arch/aarch64/Bringup.hpp>
+#include <Arch/aarch64/Gic.hpp>
+#include <Boot/BootInfo.hpp>
 #include <Libraries/Memory.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
+#include <Timekeeping/Timer.hpp>
 
 using namespace Kt;
 
@@ -47,14 +53,83 @@ namespace Smp {
             << bsp.arch.mpidr << ")";
     }
 
-    void BootAPs(const montauk::boot::SmpInfo& smp) {
-        if (smp.cpuCount > 1) {
-            g_detectedCpuCount = (int)smp.cpuCount;
+    // A secondary CPU, woken by the bootloader on its own stack and still
+    // on the bootloader's translation tables. The kernel stashed this CPU's
+    // CpuData in extraArgument.
+    static void ApEntry(montauk::boot::BootCpu* bootCpu) {
+        CpuData* cpu = (CpuData*)bootCpu->extraArgument;
+
+        Arch::InitializeCpuRegisters();
+        Arch::ActivateKernelTranslation();
+        asm volatile("msr tpidr_el1, %0" :: "r"(cpu));
+
+        Arch::Gic::InitializeCpuInterface();
+        Arch::InitializeSyscallEntry();
+        Timekeeping::TimerInitializeAP();
+
+        __atomic_store_n(&cpu->started, true, __ATOMIC_RELEASE);
+        Arch::EnableInterrupts();
+
+        // Runnable work arrives as a reschedule IPI; keep the periodic tick
+        // masked while idle.
+        for (;;) {
+            Timekeeping::TimerEnterApIdle();
+            Timekeeping::ServiceDeferredWork();
+            Arch::DisableInterrupts();
+            Arch::HaltWithInterruptsDisabled();
+            Arch::EnableInterrupts();
+        }
     }
 
-        // TODO(aarch64): start secondaries through smp.startCpu once the GIC
-        // (reschedule/TLB-shootdown SGIs) and the per-CPU generic timer work.
-        KernelLogStream(WARNING, "SMP") << "aarch64: secondary CPUs not started ("
-            << base::dec << smp.cpuCount << " reported)";
+    void BootAPs(const montauk::boot::SmpInfo& smp) {
+        if (smp.cpus == nullptr || smp.startCpu == nullptr || smp.cpuCount <= 1) {
+            KernelLogStream(INFO, "SMP") << "Single CPU system - no secondary CPUs to start";
+            return;
+        }
+        g_detectedCpuCount = (int)smp.cpuCount;
+
+        if (!Arch::Gic::IsInitialized()) {
+            KernelLogStream(WARNING, "SMP") << "No interrupt controller: secondary CPUs left parked";
+            return;
+        }
+
+        int apIndex = 1;
+        for (uint64_t i = 0; i < smp.cpuCount && apIndex < MaxCPUs; i++) {
+            montauk::boot::BootCpu& info = smp.cpus[i];
+            if (info.hardwareId == smp.bspHardwareId) continue;
+
+            CpuData& ap = g_cpus[apIndex];
+            ap.selfPtr = (uint64_t)&ap;
+            ap.cpuIndex = apIndex;
+            ap.currentSlot = -1;
+            ap.started = false;
+            ap.arch.mpidr = info.hardwareId;
+
+            info.extraArgument = (uint64_t)&ap;
+            // Publish the CpuData before the CPU that reads it is woken.
+            asm volatile("dsb ish" ::: "memory");
+            smp.startCpu(&info, ApEntry);
+            apIndex++;
+        }
+        g_cpuCount = apIndex;
+
+        int online = 1;
+        for (int i = 1; i < g_cpuCount; i++) {
+            uint64_t start = Arch::ReadTimestamp();
+            uint64_t frequency;
+            asm volatile("mrs %0, cntfrq_el0" : "=r"(frequency));
+            while (!__atomic_load_n(&g_cpus[i].started, __ATOMIC_ACQUIRE) &&
+                   Arch::ReadTimestamp() - start < frequency) {
+                Arch::Pause();
+            }
+            if (g_cpus[i].started) {
+                online++;
+            } else {
+                KernelLogStream(ERROR, "SMP") << "CPU " << base::dec << (uint64_t)i
+                    << " (MPIDR " << base::hex << g_cpus[i].arch.mpidr << ") failed to start";
+            }
+        }
+
+        KernelLogStream(OK, "SMP") << base::dec << (uint64_t)online << " CPU(s) online";
     }
 }

kernel/src/Arch/aarch64/SyscallFrame.hpp

@@ -9,13 +9,17 @@
 
 namespace Arch {
 
-    // Saved by the EL0 synchronous-exception vector for `svc #0`:
-    // x0-x5 carry the arguments and x8 the system-call number.
+    // The exception frame Vectors.S saves for `svc #0` (the same layout as
+    // Arch::PanicFrame): x0-x5 carry the arguments and x8 the system-call
+    // number; the result is written back to x0.
     struct SyscallFrame {
         uint64_t arg1, arg2, arg3, arg4, arg5, arg6;  // x0-x5
+        uint64_t x6, x7;
         uint64_t syscall_nr;                           // x8
-        uint64_t elr, spsr, sp_el0;                    // user PC, PSTATE, SP
-        uint64_t x19_x30[12];                          // callee-saved + LR
+        uint64_t x9_x30[22];
+        uint64_t sp_el0, elr, spsr, esr, far;
     };
 
+    static_assert(sizeof(SyscallFrame) == 288, "must match Vectors.S");
+
 }

kernel/src/Arch/aarch64/Timer.cpp

@@ -1,24 +1,116 @@
 /*
     * Timer.cpp
-    * aarch64 tick timer (implements Arch/Timer.hpp)
+    * aarch64 tick timer: the EL1 virtual generic timer (implements Arch/Timer.hpp)
     * Copyright (c) 2026 Daniel Hammer
 */
 
 #include <Arch/Timer.hpp>
+#include <Arch/Interrupts.hpp>
+#include <Arch/Cpu.hpp>
+#include <Arch/Smp.hpp>
+#include <Arch/aarch64/Gic.hpp>
+#include <ACPI/ACPI.hpp>
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
 
-// TODO(aarch64): drive the EL1 virtual timer (CNTV_TVAL/CNTV_CTL) from
-// CNTFRQ_EL0, re-arming on each tick, once the GIC delivers its PPI.
+// The generic timer counts at CNTFRQ_EL0 on every CPU, so it needs no
+// calibration. Each CPU's virtual timer raises a PPI (the GTDT names it,
+// normally INTID 27); the tick is re-armed from the interrupt's ack hook,
+// which also drops the level-triggered line before the EOI.
+
+using namespace Kt;
 
 namespace Arch::Timer {
 
-    bool InitializeBsp(uint32_t) {
-        return false;
+    // CNTV_CTL_EL0
+    static constexpr uint64_t CtlEnable = 1ULL << 0;
+    static constexpr uint64_t CtlMask   = 1ULL << 1;
+
+    // GTDT timer flags
+    static constexpr uint32_t GtdtEdgeTriggered = 1u << 0;
+
+    static constexpr uint32_t DefaultVirtualTimerIntid = 27;
+
+    struct Gtdt {
+        Hal::ACPI::CommonSDTHeader Header;
+        uint64_t CntControlBase;
+        uint32_t Reserved;
+        uint32_t SecureEl1Gsiv;
+        uint32_t SecureEl1Flags;
+        uint32_t NonSecureEl1Gsiv;
+        uint32_t NonSecureEl1Flags;
+        uint32_t VirtualTimerGsiv;
+        uint32_t VirtualTimerFlags;
+    } __attribute__((packed));
+
+    static uint64_t g_ticksPerMs = 0;
+    static uint32_t g_intid = DefaultVirtualTimerIntid;
+    static bool g_edgeTriggered = false;
+
+    static inline void WriteCtl(uint64_t value) {
+        asm volatile("msr cntv_ctl_el0, %0\n\tisb" :: "r"(value) : "memory");
     }
 
-    void StartPeriodic(uint32_t) {
+    static inline uint64_t ReadCval() {
+        uint64_t value;
+        asm volatile("mrs %0, cntv_cval_el0" : "=r"(value));
+        return value;
+    }
+
+    static inline void WriteCval(uint64_t value) {
+        asm volatile("msr cntv_cval_el0, %0\n\tisb" :: "r"(value) : "memory");
+    }
+
+    // Advance the deadline by one period from the previous one (no drift),
+    // skipping ticks the CPU missed rather than replaying them.
+    static void Rearm() {
+        uint64_t period = Smp::GetCurrentCpuData()->arch.timerPeriod;
+        if (period == 0) {
+            WriteCtl(CtlMask);
+            return;
+        }
+        uint64_t now = ReadTimestamp();
+        uint64_t next = ReadCval() + period;
+        if ((int64_t)(next - now) <= 0) next = now + period;
+        WriteCval(next);
+    }
+
+    static void ReadGtdt() {
+        auto* gtdt = (Gtdt*)Gic::FindTable(Gic::CurrentXsdt(), "GTDT");
+        if (gtdt == nullptr || gtdt->Header.Length < sizeof(Gtdt)) return;
+        if (gtdt->VirtualTimerGsiv >= 16 && gtdt->VirtualTimerGsiv < 32) {
+            g_intid = gtdt->VirtualTimerGsiv;
+            g_edgeTriggered = (gtdt->VirtualTimerFlags & GtdtEdgeTriggered) != 0;
+        }
+    }
+
+    bool InitializeBsp(uint32_t periodMs) {
+        uint64_t frequency;
+        asm volatile("mrs %0, cntfrq_el0" : "=r"(frequency));
+        if (frequency < 1000 || !Gic::IsInitialized()) return false;
+        g_ticksPerMs = frequency / 1000;
+
+        ReadGtdt();
+        Gic::SetAckHook(IRQ_TIMER, Rearm);
+        StartPeriodic(periodMs);
+
+        KernelLogStream(OK, "Timer") << "Generic timer: " << base::dec << frequency
+            << " Hz counter, virtual timer INTID " << (uint64_t)g_intid;
+        return true;
+    }
+
+    // PPIs are banked: each CPU enables its own timer interrupt.
+    void StartPeriodic(uint32_t periodMs) {
+        Gic::RoutePpi(g_intid, IRQ_TIMER, g_edgeTriggered);
+        uint64_t period = g_ticksPerMs * periodMs;
+        Smp::GetCurrentCpuData()->arch.timerPeriod = period;
+        WriteCval(ReadTimestamp() + period);
+        WriteCtl(CtlEnable);
     }
 
     void Stop() {
+        Smp::GetCurrentCpuData()->arch.timerPeriod = 0;
+        WriteCtl(CtlMask);
     }
 
 }

kernel/src/Arch/aarch64/Vectors.S

@@ -0,0 +1,201 @@
+/*
+    Vectors.S
+    aarch64 exception vector table (VBAR_EL1)
+    Copyright (c) 2026 Daniel Hammer
+*/
+
+/*
+ * Every exception saves the same 288-byte frame on SP_EL1, laid out as
+ * Arch::PanicFrame (InterruptFrame.hpp) and Arch::SyscallFrame:
+ *
+ *     0   x0 .. x30       (x30 at 240)
+ *   248   interrupted SP  (SP_EL0 from EL0, the pre-exception SP_EL1 from EL1)
+ *   256   ELR_EL1
+ *   264   SPSR_EL1
+ *   272   ESR_EL1
+ *   280   FAR_EL1
+ *
+ * The kernel runs on SP_EL1 (SPSel = 1, see Platform.cpp). From EL0,
+ * SP_EL1 is the top of the thread's kernel stack: ArchEnterUserMode sets
+ * it before the first ERET and every return to EL0 restores it.
+ */
+
+#define FRAME_SIZE  288
+#define FRAME_SP    248
+#define FRAME_ELR   256
+#define FRAME_SPSR  264
+#define FRAME_ESR   272
+#define FRAME_FAR   280
+
+    .text
+
+.macro SAVE_FRAME el
+    sub     sp, sp, #FRAME_SIZE
+    stp     x0, x1, [sp, #0]
+    stp     x2, x3, [sp, #16]
+    stp     x4, x5, [sp, #32]
+    stp     x6, x7, [sp, #48]
+    stp     x8, x9, [sp, #64]
+    stp     x10, x11, [sp, #80]
+    stp     x12, x13, [sp, #96]
+    stp     x14, x15, [sp, #112]
+    stp     x16, x17, [sp, #128]
+    stp     x18, x19, [sp, #144]
+    stp     x20, x21, [sp, #160]
+    stp     x22, x23, [sp, #176]
+    stp     x24, x25, [sp, #192]
+    stp     x26, x27, [sp, #208]
+    stp     x28, x29, [sp, #224]
+    .if \el == 0
+    mrs     x21, sp_el0
+    .else
+    add     x21, sp, #FRAME_SIZE
+    .endif
+    stp     x30, x21, [sp, #240]
+    mrs     x22, elr_el1
+    mrs     x23, spsr_el1
+    stp     x22, x23, [sp, #FRAME_ELR]
+    mrs     x24, esr_el1
+    mrs     x25, far_el1
+    stp     x24, x25, [sp, #FRAME_ESR]
+.endm
+
+/* Interrupts are masked again before ELR/SPSR are reloaded: an IRQ taken
+   in between would overwrite them. */
+.macro RESTORE_FRAME el
+    msr     daifset, #2
+    ldp     x22, x23, [sp, #FRAME_ELR]
+    msr     elr_el1, x22
+    msr     spsr_el1, x23
+    .if \el == 0
+    ldr     x21, [sp, #FRAME_SP]
+    msr     sp_el0, x21
+    .endif
+    ldp     x0, x1, [sp, #0]
+    ldp     x2, x3, [sp, #16]
+    ldp     x4, x5, [sp, #32]
+    ldp     x6, x7, [sp, #48]
+    ldp     x8, x9, [sp, #64]
+    ldp     x10, x11, [sp, #80]
+    ldp     x12, x13, [sp, #96]
+    ldp     x14, x15, [sp, #112]
+    ldp     x16, x17, [sp, #128]
+    ldp     x18, x19, [sp, #144]
+    ldp     x20, x21, [sp, #160]
+    ldp     x22, x23, [sp, #176]
+    ldp     x24, x25, [sp, #192]
+    ldp     x26, x27, [sp, #208]
+    ldp     x28, x29, [sp, #224]
+    ldr     x30, [sp, #240]
+    add     sp, sp, #FRAME_SIZE
+    eret
+.endm
+
+/* Each vector slot is 128 bytes; the handlers live outside the table. */
+.macro VECTOR target
+    .balign 0x80
+    b       \target
+.endm
+
+    .balign 0x800
+    .global ArchExceptionVectors
+ArchExceptionVectors:
+    /* Current EL with SP_EL0: the kernel never runs on SP_EL0. */
+    VECTOR  el1t_unexpected
+    VECTOR  el1t_unexpected
+    VECTOR  el1t_unexpected
+    VECTOR  el1t_unexpected
+    /* Current EL with SP_ELx */
+    VECTOR  el1h_sync
+    VECTOR  el1h_irq
+    VECTOR  el1h_fiq
+    VECTOR  el1h_serror
+    /* Lower EL, AArch64 */
+    VECTOR  el0_sync
+    VECTOR  el0_irq
+    VECTOR  el0_fiq
+    VECTOR  el0_serror
+    /* Lower EL, AArch32: not supported */
+    VECTOR  el0_aarch32
+    VECTOR  el0_aarch32
+    VECTOR  el0_aarch32
+    VECTOR  el0_aarch32
+
+/* ---- Exceptions from the kernel ---- */
+
+el1t_unexpected:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #0
+    bl      ArchUnexpectedException
+
+el1h_sync:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #0
+    bl      ArchSyncException
+    RESTORE_FRAME 1
+
+el1h_irq:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #0
+    bl      ArchIrqException
+    RESTORE_FRAME 1
+
+el1h_fiq:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #1
+    bl      ArchUnexpectedException
+
+el1h_serror:
+    SAVE_FRAME 1
+    mov     x0, sp
+    mov     x1, #2
+    bl      ArchUnexpectedException
+
+/* ---- Exceptions from user mode ---- */
+
+el0_sync:
+    SAVE_FRAME 0
+    lsr     x9, x24, #26            /* ESR_EL1.EC */
+    cmp     x9, #0x15               /* SVC from AArch64 */
+    b.ne    1f
+
+    msr     daifclr, #2
+    mov     x0, sp
+    bl      SyscallDispatch
+    str     x0, [sp, #0]
+    bl      SchedExitIfKilled
+    RESTORE_FRAME 0
+
+1:  mov     x0, sp
+    mov     x1, #1
+    bl      ArchSyncException
+    RESTORE_FRAME 0
+
+el0_irq:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #1
+    bl      ArchIrqException
+    RESTORE_FRAME 0
+
+el0_fiq:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #1
+    bl      ArchUnexpectedException
+
+el0_serror:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #2
+    bl      ArchUnexpectedException
+
+el0_aarch32:
+    SAVE_FRAME 0
+    mov     x0, sp
+    mov     x1, #3
+    bl      ArchUnexpectedException

kernel/src/Arch/x86_64/Apic/Interrupts.cpp

@@ -58,6 +58,14 @@ namespace Arch {
         Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irq));
     }
 
+    bool PlatformLinesAreFixed() {
+        return true;
+    }
+
+    bool AssignPlatformLine(uint8_t, uint32_t, bool, bool) {
+        return false;
+    }
+
     bool ComposeMsi(uint8_t irq, MsiMessage& out) {
         // Physical destination mode, fixed delivery, edge triggered, to the
         // BSP (APIC ID 0 -- the destination field is left zero, as every

kernel/src/Arch/x86_64/Cpu.hpp

@@ -24,6 +24,15 @@ namespace Arch {
         asm volatile("mfence" ::: "memory");
     }
 
+    // Stores are not reordered with other stores, nor loads with loads.
+    inline void DmaWriteFence() {
+        asm volatile("" ::: "memory");
+    }
+
+    inline void DmaReadFence() {
+        asm volatile("" ::: "memory");
+    }
+
     inline void EnableInterrupts() {
         asm volatile("sti" ::: "memory");
     }
@@ -79,6 +88,15 @@ namespace Arch {
         asm volatile("wbinvd" ::: "memory");
     }
 
+    inline void SyncInstructionCache(const void*, uint64_t) {
+    }
+
+    inline void CleanDataCache(const void*, uint64_t) {
+    }
+
+    inline void InvalidateDataCache(const void*, uint64_t) {
+    }
+
     inline void FlushTlbPage(uint64_t virtualAddress) {
         asm volatile("invlpg (%0)" :: "r"(virtualAddress) : "memory");
     }

kernel/src/Arch/x86_64/Platform.cpp

@@ -77,4 +77,8 @@ namespace Arch {
         __builtin_unreachable();
     }
 
+    // x86 powers off through ACPI S5 (AcpiShutdown).
+    void PowerOff() {
+    }
+
 }

kernel/src/Boot/Main.cpp

@@ -27,10 +27,12 @@
 #include <Pci/Pci.hpp>
 #include <Timekeeping/Timer.hpp>
 #include <Drivers/Init.hpp>
+#include <Drivers/Serial/Uart.hpp>
 #include <Graphics/Framebuffer.hpp>
 #include <Arch/Cpu.hpp>
 #include <Arch/Platform.hpp>
 #include <Fs/Boot.hpp>
+#include <Fs/Vfs.hpp>
 #include <Sched/Scheduler.hpp>
 #include <Ipc/Ipc.hpp>
 #include <Api/Syscall.hpp>
@@ -111,6 +113,8 @@ extern "C" void kmain() {
 
     Hal::ACPI g_acpi((Hal::ACPI::XSDP*)Memory::HHDM(boot.rsdpPhysical));
 
+    Drivers::Serial::InitializeConsole(g_acpi.GetXSDT());
+
     if (g_acpi.GetXSDT() != nullptr) {
         Hal::AcpiShutdown::Initialize(g_acpi.GetXSDT());
 
@@ -135,6 +139,8 @@ extern "C" void kmain() {
         // power control on the answer.
         Hal::AML::InitializeNamespaceObjects();
 
+        Pci::AssignInterruptLines();
+
         Drivers::ProbeEarly();
         Drivers::InitializeGraphics();
 
@@ -173,6 +179,11 @@ extern "C" void kmain() {
 
     Arch::LateInitialize();
 
+    // Without a root nothing can start; say so instead of idling silently.
+    if (!Fs::Vfs::IsDriveRegistered(0)) {
+        Panic("No root filesystem (0:/): cannot start 0:/os/init.elf", nullptr);
+    }
+
     Kt::SuppressKernelLog();
     Sched::Spawn("0:/os/init.elf");
 

kernel/src/CppLib/Spinlock.hpp

@@ -36,6 +36,9 @@ namespace kcp {
                 Arch::Pause();
             }
         }
+        bool TryAcquire() {
+            return !flag.test_and_set(std::memory_order_acquire);
+        }
         void Release() {
             flag.clear(std::memory_order_release);
         }

kernel/src/Drivers/Audio/IntelHda.cpp

@@ -191,7 +191,8 @@ namespace Drivers::Audio::IntelHda {
         // Write verb to CORB
         g_corb[wp] = verb;
 
-        // Update write pointer
+        // Update write pointer once the verb is visible
+        Arch::DmaWriteFence();
         Write16(REG_CORBWP, wp);
 
         return true;
@@ -202,6 +203,7 @@ namespace Drivers::Audio::IntelHda {
             uint16_t wp = Read16(REG_RIRBWP) & 0xFF;
 
             if (g_rirbReadPtr != wp) {
+                Arch::DmaReadFence();
                 g_rirbReadPtr = (g_rirbReadPtr + 1) % g_rirbSize;
 
                 uint32_t ex = g_rirb[g_rirbReadPtr].ResponseEx;

kernel/src/Drivers/Audio/Mixer.cpp

@@ -7,9 +7,9 @@
 
 #include "Mixer.hpp"
 #include "IntelHda.hpp"
-#include <Drivers/USB/Bluetooth/Bluetooth.hpp>
-#include <Drivers/USB/Bluetooth/A2dp.hpp>
-#include <Drivers/USB/Bluetooth/Avrcp.hpp>
+#include <Drivers/Bluetooth/Bluetooth.hpp>
+#include <Drivers/Bluetooth/A2dp.hpp>
+#include <Drivers/Bluetooth/Avrcp.hpp>
 #include <Memory/PageFrameAllocator.hpp>
 #include <Memory/HHDM.hpp>
 #include <Sched/Scheduler.hpp>
@@ -146,8 +146,8 @@ namespace Drivers::Audio::Mixer {
     }
 
     static bool BluetoothReady() {
-        if (!Drivers::USB::Bluetooth::IsInitialized()) return false;
-        return Drivers::USB::Bluetooth::A2dp::IsReady();
+        if (!Drivers::Bluetooth::IsInitialized()) return false;
+        return Drivers::Bluetooth::A2dp::IsReady();
     }
 
     static void SyncHdaMasterMute() {
@@ -236,7 +236,7 @@ namespace Drivers::Audio::Mixer {
     static uint32_t BackendWriteSpace() {
         if (g_output == Output::Bluetooth) {
             return g_btOpened
-                ? Drivers::USB::Bluetooth::A2dp::GetWriteSpace() : 0;
+                ? Drivers::Bluetooth::A2dp::GetWriteSpace() : 0;
         }
         return g_hdaOpened ? IntelHda::GetWriteSpace(g_hdaHandle) : 0;
     }
@@ -244,7 +244,7 @@ namespace Drivers::Audio::Mixer {
     static int BackendWrite(const uint8_t* data, uint32_t size) {
         if (g_output == Output::Bluetooth) {
             return g_btOpened
-                ? Drivers::USB::Bluetooth::A2dp::WriteAudio(data, size) : -1;
+                ? Drivers::Bluetooth::A2dp::WriteAudio(data, size) : -1;
         }
         return g_hdaOpened ? IntelHda::Write(g_hdaHandle, data, size) : -1;
     }
@@ -403,14 +403,14 @@ namespace Drivers::Audio::Mixer {
         if (g_output == Output::Bluetooth && !g_btOpened && BluetoothReady()) {
             g_switchingOutput = true;
             g_lock.Release();
-            bool btOk = Drivers::USB::Bluetooth::A2dp::ConfigureStream(
+            bool btOk = Drivers::Bluetooth::A2dp::ConfigureStream(
                             MIX_RATE, MIX_CHANNELS, MIX_BITS) &&
-                        Drivers::USB::Bluetooth::A2dp::StartStream();
+                        Drivers::Bluetooth::A2dp::StartStream();
             g_lock.Acquire();
             g_btOpened = btOk;
             if (!btOk) g_output = Output::Hda;
             if (btOk)
-                Drivers::USB::Bluetooth::A2dp::SetMuted(
+                Drivers::Bluetooth::A2dp::SetMuted(
                     g_masterMute.load(std::memory_order_acquire));
             g_switchingOutput = false;
             wakeSwitchWaiters = true;
@@ -515,7 +515,7 @@ namespace Drivers::Audio::Mixer {
         if (closeBt) g_btOpened = false;
         g_lock.Release();
         if (closeHda) IntelHda::Close(hdaHandle);
-        if (closeBt) Drivers::USB::Bluetooth::A2dp::StopStream(true);
+        if (closeBt) Drivers::Bluetooth::A2dp::StopStream(true);
         if (ringToFree) FreeRing(ringToFree);
         if (changed) Sched::WakeObjectWaiters((void*)&g_serial);
     }
@@ -535,7 +535,7 @@ namespace Drivers::Audio::Mixer {
         bool serviceBluetooth = g_output == Output::Bluetooth;
         g_lock.Release();
         if (serviceBluetooth)
-            Drivers::USB::Bluetooth::A2dp::ServiceMedia();
+            Drivers::Bluetooth::A2dp::ServiceMedia();
         return (int)written;
     }
 
@@ -651,7 +651,7 @@ namespace Drivers::Audio::Mixer {
         if (changed) BumpSerialLocked();
         g_lock.Release();
         if (closeHda) IntelHda::Close(hdaHandle);
-        if (closeBt) Drivers::USB::Bluetooth::A2dp::StopStream(true);
+        if (closeBt) Drivers::Bluetooth::A2dp::StopStream(true);
         for (int i = 0; i < ringCount; i++) FreeRing(ringsToFree[i]);
         if (changed) Sched::WakeObjectWaiters((void*)&g_serial);
     }
@@ -668,7 +668,7 @@ namespace Drivers::Audio::Mixer {
         g_lock.Release();
 
         if (changed)
-            Drivers::USB::Bluetooth::Avrcp::NotifyVolumeChanged(percent);
+            Drivers::Bluetooth::Avrcp::NotifyVolumeChanged(percent);
         if (changed) Sched::WakeObjectWaiters((void*)&g_serial);
     }
 
@@ -688,7 +688,7 @@ namespace Drivers::Audio::Mixer {
 
         if (changed) SyncHdaMasterMute();
         if (changed)
-            Drivers::USB::Bluetooth::A2dp::SetMuted(muted);
+            Drivers::Bluetooth::A2dp::SetMuted(muted);
         if (changed) Sched::WakeObjectWaiters((void*)&g_serial);
     }
 
@@ -733,11 +733,11 @@ namespace Drivers::Audio::Mixer {
             }
         } else if (needBackend && output == Output::Bluetooth) {
             prepared = BluetoothReady() &&
-                       Drivers::USB::Bluetooth::A2dp::ConfigureStream(
+                       Drivers::Bluetooth::A2dp::ConfigureStream(
                            MIX_RATE, MIX_CHANNELS, MIX_BITS) &&
-                       Drivers::USB::Bluetooth::A2dp::StartStream();
+                       Drivers::Bluetooth::A2dp::StartStream();
             if (prepared)
-                Drivers::USB::Bluetooth::A2dp::SetMuted(
+                Drivers::Bluetooth::A2dp::SetMuted(
                     g_masterMute.load(std::memory_order_acquire));
         } else if (output == Output::Hda) {
             prepared = IntelHda::IsInitialized();
@@ -779,7 +779,7 @@ namespace Drivers::Audio::Mixer {
         Sched::WakeObjectWaiters((void*)&g_switchingOutput);
 
         if (closeHda) IntelHda::Close(oldHdaHandle);
-        if (closeBt) Drivers::USB::Bluetooth::A2dp::StopStream(true);
+        if (closeBt) Drivers::Bluetooth::A2dp::StopStream(true);
         Sched::WakeObjectWaiters((void*)&g_serial);
         return 0;
     }
@@ -815,7 +815,7 @@ namespace Drivers::Audio::Mixer {
             wake = true;
         }
         g_lock.Release();
-        if (stopBt) Drivers::USB::Bluetooth::A2dp::StopStream(true);
+        if (stopBt) Drivers::Bluetooth::A2dp::StopStream(true);
         if (wake) Sched::WakeObjectWaiters((void*)&g_serial);
     }
 

kernel/src/Drivers/Bluetooth/A2dp.cpp

@@ -9,7 +9,6 @@
 #include "Sbc.hpp"
 #include "L2cap.hpp"
 #include "Hci.hpp"
-#include <Drivers/USB/Xhci.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 #include <Libraries/Memory.hpp>
@@ -18,7 +17,7 @@
 
 using namespace Kt;
 
-namespace Drivers::USB::Bluetooth::A2dp {
+namespace Drivers::Bluetooth::A2dp {
 
     // =========================================================================
     // AVDTP constants
@@ -222,7 +221,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
         uint64_t start = Timekeeping::GetMilliseconds();
 
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();   // AVDTP responses arrive as ACL data
             if (g_avdtpResponseReady.load(std::memory_order_acquire)) {
                 // Freeze this mailbox before returning to the caller.  A late
@@ -557,7 +556,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
 
     void OnChannelReady(uint16_t l2capCid) {
         // Invoked from L2CAP inside the ACL receive path -- i.e. NESTED under
-        // Xhci::PollEvents.  PollEvents is non-reentrant (re-entry guard), so we
+        // Hci::DrainEvents.  That is non-reentrant (re-entry guard), so we
         // must NOT run the blocking AVDTP signaling chain here; a nested poll
         // would stall and every WaitAvdtpResponse() would time out.  Just record
         // the channel; StartSource() drives the handshakes from top-level
@@ -1161,7 +1160,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
         bool answered = false;
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();
             if (g_sdpRspReady.load(std::memory_order_acquire)) {
                 answered = true;
@@ -1196,7 +1195,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
     static bool WaitEncrypted(uint32_t timeoutMs) {
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();
             Hci::ProcessPendingCommands();
             auto* c = Hci::GetConnection(g_aclHandle.load(std::memory_order_acquire));
@@ -1235,7 +1234,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
             uint64_t waitStart = Timekeeping::GetMilliseconds();
             while (g_sourceSetupActive.load(std::memory_order_acquire)
                    && Timekeeping::GetMilliseconds() - waitStart < timeoutMs) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 for (int j = 0; j < 100; j++) Arch::Pause();
             }
@@ -1265,7 +1264,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
         if (g_sigCid.load(std::memory_order_acquire) == 0) {
             uint64_t lStart = Timekeeping::GetMilliseconds();
             while (Timekeeping::GetMilliseconds() - lStart < 1000) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 Hci::ProcessPendingCommands();
                 if (g_sigCid.load(std::memory_order_acquire) != 0) break;
@@ -1319,7 +1318,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
             uint32_t attemptWaitMs = timeoutMs < 1500 ? timeoutMs : 1500;
             uint32_t waitBudgetMs = attemptWaitMs;
             while (Timekeeping::GetMilliseconds() - sigStart < waitBudgetMs) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 if (g_sigCid.load(std::memory_order_acquire) != 0) break;
                 // PENDING is explicitly non-final.  A peer performing service
@@ -1363,7 +1362,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
             if (sig) L2cap::FreeChannel(sig);
             uint64_t st = Timekeeping::GetMilliseconds();
             while (Timekeeping::GetMilliseconds() - st < 400) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 for (int j = 0; j < 200; j++) Arch::Pause();
             }
@@ -1384,7 +1383,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
         if (g_peerConfigured.load(std::memory_order_acquire)) {
             uint64_t peerStart = Timekeeping::GetMilliseconds();
             while (Timekeeping::GetMilliseconds() - peerStart < timeoutMs) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 State s = g_state.load(std::memory_order_acquire);
                 if (s == State::Open || s == State::Streaming) break;
@@ -1483,7 +1482,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
         uint64_t mStart = Timekeeping::GetMilliseconds();
         while (g_mediaCid.load(std::memory_order_acquire) == 0
                && Timekeeping::GetMilliseconds() - mStart < kMediaWaitMs) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();
             auto* ch = media ? L2cap::GetChannel(media) : nullptr;
             if (ch && ch->Configured) {
@@ -1838,6 +1837,36 @@ namespace Drivers::USB::Bluetooth::A2dp {
     // ConfigureStream
     // =========================================================================
 
+    // A UART transport has a hard byte budget: at 460800 baud it carries
+    // 46 KB/s, and SBC at bitpool 53 / 48 kHz needs ~46 KB/s with its
+    // headers, so the stream fell behind whenever anything else crossed the
+    // link and the sink played concealment. Keep the stream within three
+    // quarters of the line rate by lowering the bitpool (any value in the
+    // negotiated range is valid per frame). Over USB nothing changes.
+    static void FitBitpoolToTransport(uint32_t sampleRate) {
+        uint32_t budget = Hci::TransportBytesPerSecond() / 4 * 3;
+        if (budget == 0) return;
+        uint32_t rate = sampleRate ? sampleRate : 48000;
+        uint8_t minBitpool = g_cfgSbc[2] < 2 ? 2 : g_cfgSbc[2];
+        uint8_t original = g_sbcEncoder.Bitpool;
+        auto bytesPerSecond = [&]() {
+            // H4 + ACL + L2CAP + RTP + SCMS-T + SBC headers (23 bytes) per
+            // packet, spread over the ~4-5 frames bundled into each.
+            uint32_t frameCost = Sbc::GetFrameSize(&g_sbcEncoder) + 6;
+            return (uint64_t)frameCost * rate / Sbc::GetSamplesPerFrame(&g_sbcEncoder);
+        };
+        while (g_sbcEncoder.Bitpool > minBitpool && bytesPerSecond() > budget) {
+            Sbc::Configure(&g_sbcEncoder, g_cfgSbc[0], g_cfgSbc[1],
+                           (uint8_t)(g_sbcEncoder.Bitpool - 1));
+        }
+        if (g_sbcEncoder.Bitpool != original) {
+            KernelLogStream(INFO, "BT-A2DP") << "Bitpool " << (uint64_t)original << " -> "
+                << (uint64_t)g_sbcEncoder.Bitpool << " to fit the transport ("
+                << bytesPerSecond() << " of " << (uint64_t)Hci::TransportBytesPerSecond()
+                << " bytes/s)";
+        }
+    }
+
     bool ConfigureStream(uint32_t sampleRate, uint8_t channels, uint8_t bitsPerSample) {
         // Encoder configuration and PumpMedia both mutate the SBC encoder.
         // Device switching normally configures an Open stream, but explicitly
@@ -1852,6 +1881,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
         // agreed to (Init's defaults may differ from the negotiated subset).
         if (g_cfgSbc[0] || g_cfgSbc[1]) {
             Sbc::Configure(&g_sbcEncoder, g_cfgSbc[0], g_cfgSbc[1], g_cfgSbc[3]);
+            FitBitpoolToTransport(sampleRate);
         }
         g_sbcInitialized = true;
         g_seqNum = 0;
@@ -2136,7 +2166,7 @@ namespace Drivers::USB::Bluetooth::A2dp {
 
     void ServiceMedia() {
         if (!AcquireMediaService()) return;
-        Xhci::PollEvents();
+        Hci::Pump();
         Hci::DrainEvents();
         PumpMedia();
         g_serviceActive.store(false, std::memory_order_release);

kernel/src/Drivers/Bluetooth/A2dp.hpp

@@ -8,7 +8,7 @@
 #pragma once
 #include <cstdint>
 
-namespace Drivers::USB::Bluetooth::A2dp {
+namespace Drivers::Bluetooth::A2dp {
 
     // =========================================================================
     // A2DP stream states

kernel/src/Drivers/Bluetooth/Avrcp.cpp

@@ -15,7 +15,7 @@
 
 using namespace Kt;
 
-namespace Drivers::USB::Bluetooth::Avrcp {
+namespace Drivers::Bluetooth::Avrcp {
 
     // =========================================================================
     // Constants

kernel/src/Drivers/Bluetooth/Avrcp.hpp

@@ -7,7 +7,7 @@
 #pragma once
 #include <cstdint>
 
-namespace Drivers::USB::Bluetooth::Avrcp {
+namespace Drivers::Bluetooth::Avrcp {
 
     // Process an AVCTP packet arriving on an AVRCP control channel (PSM 0x0017).
     // Implements a minimal AVRCP Target: unit/subunit info, honest rejection of

kernel/src/Drivers/Bluetooth/Bluetooth.cpp

@@ -1,6 +1,6 @@
 /*
     * Bluetooth.cpp
-    * Top-level Bluetooth subsystem — adapter registration and Intel BT initialization
+    * Top-level Bluetooth subsystem — adapter registration and controller bring-up
     * Copyright (c) 2026 Daniel Hammer
 */
 
@@ -9,6 +9,7 @@
 #include "Hci.hpp"
 #include "A2dp.hpp"
 #include "IntelFirmware.hpp"
+#include "Broadcom.hpp"
 #include <Drivers/Audio/Mixer.hpp>
 #include <Drivers/USB/Xhci.hpp>
 #include <Drivers/USB/UsbDevice.hpp>
@@ -22,7 +23,9 @@
 
 using namespace Kt;
 
-namespace Drivers::USB::Bluetooth {
+namespace Drivers::Bluetooth {
+
+    namespace Xhci = Drivers::USB::Xhci;
 
     // =========================================================================
     // State
@@ -205,7 +208,9 @@ namespace Drivers::USB::Bluetooth {
     // =========================================================================
 
     void RegisterAdapter(uint8_t slotId) {
-        if (g_initialized) {
+        // One adapter at a time, whichever came first (an on-board UART
+        // controller may still be finishing its deferred init).
+        if (g_initialized || Hci::GetTransport() != Hci::Transport::None) {
             KernelLogStream(WARNING, "BT") << "Bluetooth adapter already registered";
             return;
         }
@@ -249,17 +254,47 @@ namespace Drivers::USB::Bluetooth {
         CompleteInit();
     }
 
+    // =========================================================================
+    // ProbePlatformAdapter — an on-board UART controller ACPI describes
+    // =========================================================================
+
+    bool ProbePlatformAdapter() {
+        if (g_initialized || Hci::GetTransport() != Hci::Transport::None) return false;
+        if (!Broadcom::ProbePlatform()) return false;
+
+        // The firmware patch lives on the root filesystem, which the boot
+        // driver probe precedes; finish from the idle loop like USB adapters.
+        if (!Fs::Vfs::IsDriveRegistered(0)) {
+            g_initPending.store(true, std::memory_order_release);
+            KernelLogStream(INFO, "BT") << "Transport up; deferring init until the root is mounted";
+            return true;
+        }
+        CompleteInit();
+        return true;
+    }
+
     // =========================================================================
     // CompleteInit — firmware-dependent HCI bring-up (needs VFS/ramdisk)
     // =========================================================================
 
     static void CompleteInit() {
-        auto* dev = Xhci::GetDevice(g_slotId);
-        if (!dev) return;
+        bool didReset = false;
+
+        if (Hci::GetTransport() == Hci::Transport::Uart) {
+            // Power cycle, firmware patch and operating baud; ends with a reset.
+            if (!Broadcom::Initialize()) {
+                KernelLogStream(ERROR, "BT") << "Broadcom BT initialization failed";
+                return;
+            }
+            didReset = true;
+        }
+
+        auto* dev = Hci::GetTransport() == Hci::Transport::Usb ? Xhci::GetDevice(g_slotId) : nullptr;
+        if (Hci::GetTransport() == Hci::Transport::Usb && !dev) return;
 
         // Intel-specific initialization (firmware download + HCI Reset)
-        bool didReset = false;
-        if (IsIntelBt(dev->VendorId, dev->ProductId)) {
+        if (dev && IsIntelBt(dev->VendorId, dev->ProductId)) {
+            Hci::SetVendor(Hci::Vendor::Intel);
             if (InitIntelBluetooth()) {
                 didReset = true;  // InitIntelBluetooth already sent HCI Reset
             } else {
@@ -280,9 +315,11 @@ namespace Drivers::USB::Bluetooth {
             }
         }
 
-        // Apply a persisted BD_ADDR override (0:/config/bluetooth.toml) now,
-        // after the last reset and before the address is read back, so the
-        // whole bring-up below uses the overridden address.
+        // A Broadcom part still on its chip-wide default address gets the
+        // board's, then a persisted BD_ADDR override (0:/config/bluetooth.toml)
+        // wins over both.  Both after the last reset and before the address
+        // is read back, so the whole bring-up below uses the final address.
+        if (Hci::GetVendor() == Hci::Vendor::Broadcom) Broadcom::FixDefaultAddress();
         ApplyConfiguredAddress();
 
         // Read BD_ADDR
@@ -387,7 +424,7 @@ namespace Drivers::USB::Bluetooth {
     void ServiceDeferredInit() {
         if (!g_initPending.load(std::memory_order_relaxed) || g_initialized) return;
         if (!Fs::Vfs::IsDriveRegistered(0)) return;   // ramdisk still not mounted
-        if (Xhci::InPollContext()) return;            // never nest under PollEvents
+        if (Hci::InPumpContext()) return;            // never nest under the transport pump
 
         // Claim the pending init (this runs from the idle loop; make sure only
         // one pass performs the bring-up).  The firmware download inside takes
@@ -425,12 +462,12 @@ namespace Drivers::USB::Bluetooth {
             !g_initialized && Fs::Vfs::IsDriveRegistered(0)) {
             return true;
         }
-        return g_initialized && Hci::HasPendingCommands();
+        return g_initialized && (Hci::HasPendingCommands() || Hci::TransportHasRxData());
     }
 
     void ServiceEvents() {
         if (!g_initialized) return;
-        if (Xhci::InPollContext()) return;   // never nest under PollEvents
+        if (Hci::InPumpContext()) return;   // never nest under the transport pump
         A2dp::ServiceMedia(); // reap events and feed queued media
         // DrainEvents may just have queued an accept/auth/encryption reply.
         // Send it in this same service pass instead of adding a scheduler-turn
@@ -492,7 +529,7 @@ namespace Drivers::USB::Bluetooth {
             uint64_t t0 = Timekeeping::GetMilliseconds();
             bool anyActive = true;
             while (anyActive && Timekeeping::GetMilliseconds() - t0 < 2000) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 anyActive = false;
                 for (int i = 0; i < Hci::MAX_CONNECTIONS; i++) {
@@ -548,7 +585,7 @@ namespace Drivers::USB::Bluetooth {
         // Poll until inquiry completes or timeout
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Hci::IsInquiryActive() && (Timekeeping::GetMilliseconds() - start < timeoutMs)) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();
 
             for (int j = 0; j < 200; j++) {
@@ -566,7 +603,7 @@ namespace Drivers::USB::Bluetooth {
                 uint64_t cancelStart = Timekeeping::GetMilliseconds();
                 while (Hci::IsInquiryActive()
                        && Timekeeping::GetMilliseconds() - cancelStart < 250) {
-                    Xhci::PollEvents();
+                    Hci::Pump();
                     Hci::DrainEvents();
                     for (int j = 0; j < 100; j++)
                         Arch::Pause();
@@ -612,7 +649,7 @@ namespace Drivers::USB::Bluetooth {
         // Wait for Connection Complete event
         uint64_t start = Timekeeping::GetMilliseconds();
         while (!target && Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();
             target = FindAclConnection(bdAddr);
             for (int j = 0; j < 200; j++) {
@@ -631,7 +668,7 @@ namespace Drivers::USB::Bluetooth {
             if (reusedAcl) Hci::AuthenticateLink(target->Handle);
             uint64_t t0 = Timekeeping::GetMilliseconds();
             while (Timekeeping::GetMilliseconds() - t0 < 5000) {
-                Xhci::PollEvents();
+                Hci::Pump();
                 Hci::DrainEvents();
                 Hci::ProcessPendingCommands();
                 if (!target->Active || !SameAddress(target->BdAddr, bdAddr)) break;

kernel/src/Drivers/Bluetooth/Bluetooth.hpp

@@ -8,28 +8,35 @@
 #include <cstdint>
 #include "Hci.hpp"
 
-namespace Drivers::USB::Bluetooth {
+namespace Drivers::Bluetooth {
 
     // Called by USB enumeration when a Bluetooth adapter is detected
     void RegisterAdapter(uint8_t slotId);
 
+    // Called by the boot driver probe: bring up an on-board UART controller
+    // ACPI describes (the Raspberry Pi 4/400's Broadcom radio).  Returns true
+    // if one was found; its firmware-dependent init is deferred like USB's.
+    bool ProbePlatformAdapter();
+
     // Complete any firmware-dependent bring-up that was deferred because the
     // adapter enumerated during the boot port scan, before the ramdisk (which
     // holds the firmware images) was mounted.  Safe to call unconditionally
     // once the boot filesystems are up; a no-op if nothing is pending.
     void ServiceDeferredInit();
 
-    // Steady-state event service: drains the xHCI event ring + Bluetooth ACL
-    // RX ring and flushes queued HCI replies.  Called from the idle loop so
+    // Steady-state event service: drains the transport (xHCI event ring or
+    // UART receive ring) + Bluetooth ACL RX ring and flushes queued HCI
+    // replies.  Called from the idle loop so
     // inbound traffic (the headset's SDP/AVRCP queries, AVDTP commands, ACL
     // credits) is processed even when no syscall happens to be pumping a BT
     // wait loop.  Cheap no-op when the adapter is down; safe from any core
-    // (PollEvents/DrainEvents/ProcessPendingCommands all self-serialize).
+    // (Pump/DrainEvents/ProcessPendingCommands all self-serialize).
     void ServiceEvents();
 
-    // True when boot-deferred initialization or queued HCI control work needs
-    // an idle-context service pass. USB receive events are signaled separately
-    // by xHCI and cause the dispatcher to service Bluetooth in the same pass.
+    // True when boot-deferred initialization, queued HCI control work or
+    // received UART bytes need an idle-context service pass. USB receive
+    // events are signaled separately by xHCI and cause the dispatcher to
+    // service Bluetooth in the same pass.
     bool HasDeferredWork();
 
     // Query adapter state
@@ -40,8 +47,8 @@ namespace Drivers::USB::Bluetooth {
     const uint8_t* GetBdAddr();
 
     // Change the adapter's BD_ADDR at runtime.  Drops any active links (their
-    // pairing/crypto was bound to the old address), issues the Intel vendor
-    // write (0xFC31), re-reads the controller's address into the cache and
+    // pairing/crypto was bound to the old address), issues the vendor write
+    // (Intel 0xFC31, Broadcom 0xFC01), re-reads the controller's address into the cache and
     // re-asserts scan.  Returns true only if the controller reports the new
     // address.  Persisting the choice to 0:/config/bluetooth.toml is the
     // caller's job (userspace); the override is re-applied on boot by

kernel/src/Drivers/Bluetooth/Broadcom.cpp

@@ -0,0 +1,555 @@
+/*
+    * Broadcom.cpp
+    * Broadcom/Cypress UART Bluetooth controller bring-up (Raspberry Pi 4/400)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Broadcom.hpp"
+#include "Hci.hpp"
+#include <ACPI/AcpiDevices.hpp>
+#include <ACPI/AML/AmlInterpreter.hpp>
+#include <Arch/Cpu.hpp>
+#include <CppLib/Stream.hpp>
+#include <Drivers/Platform/RpiFirmware.hpp>
+#include <Drivers/Platform/RpiGpio.hpp>
+#include <Drivers/Serial/BtUart.hpp>
+#include <Drivers/Serial/Uart.hpp>
+#include <Fs/Vfs.hpp>
+#include <Libraries/Memory.hpp>
+#include <Memory/Heap.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// The Raspberry Pi 4B (CYW43455) and 400 (CYW43456) carry a Cypress radio
+// whose Bluetooth half speaks HCI over a UART (H4) on GPIO 30-33: CTS, RTS,
+// TXD, RXD. Its power switch, BT_ON, is line 0 of the VideoCore firmware's
+// GPIO expander.
+//
+// Bring-up follows Linux's hci_bcm + btbcm:
+//   1. power on, wait ~100 ms, HCI Reset at the controller's 115200 baud;
+//   2. Read Local Version: the LMP subversion names the chip and so the
+//      patch file (BCM4345C0.hcd, BCM4345C5.hcd);
+//   3. raise both ends to the operating speed (vendor 0xFC18, answered at
+//      the old speed);
+//   4. Download Minidriver (0xFC2E), wait 50 ms, then send every record of
+//      the .hcd file -- each one is a complete HCI command, Write RAM
+//      (0xFC4C) and finally Launch RAM (0xFC4E) -- and wait 250 ms;
+//   5. the patched firmware restarts at 115200: HCI Reset there, and raise
+//      the speed again.
+//
+// Raspberry Pi OS's btuart runs the controller at 460800 without flow
+// control on the mini UART and at 3 Mbaud with it on the PL011. 460800 is
+// too slow for A2DP here: 8N1 carries 46 KB/s and SBC at bitpool 53 needs
+// ~46 KB/s with its headers, so music broke up whenever anything else used
+// the link. The mini UART's auto RTS/CTS allows 921600, so it is tried
+// first; without flow control (460800) is the fallback, and A2DP lowers its
+// bitpool to fit (Hci::TransportBytesPerSecond).
+//
+// Our RTS must be deasserted while the controller powers on: with it
+// asserted the controller boots into autobaud mode (see PowerCycle).
+
+using namespace Kt;
+
+namespace Drivers::Bluetooth::Broadcom {
+
+    namespace BtUart = Drivers::Serial::BtUart;
+    namespace RpiFirmware = Drivers::Platform::RpiFirmware;
+    namespace RpiGpio = Drivers::Platform::RpiGpio;
+
+    // GPIO 30-33 carry CTS, RTS, TXD, RXD: alternate function 3 routes them
+    // to the PL011, 5 to the mini UART. Pulls as Linux's bcm2711 pinctrl
+    // sets them (uart0_ctsrts_gpio30, uart0_gpio32).
+    static constexpr uint32_t PIN_CTS = 30;
+    static constexpr uint32_t PIN_RTS = 31;
+    static constexpr uint32_t PIN_TXD = 32;
+    static constexpr uint32_t PIN_RXD = 33;
+
+    static constexpr uint32_t InitialBaud = 115200;
+
+    // LMP subversions of the ROM firmware -> patch file (Linux btbcm's
+    // bcm_uart_subver_table).
+    struct ChipName {
+        uint16_t    subversion;
+        const char* name;
+    };
+    static constexpr ChipName g_chips[] = {
+        { 0x6119, "BCM4345C0" },   // CYW43455: Raspberry Pi 3B+, 4B
+        { 0x6606, "BCM4345C5" },   // CYW43456: Raspberry Pi 400, CM4
+    };
+
+    static bool g_flowControl = false;
+    static bool g_powerSwitch = true;
+
+    // =========================================================================
+    // Platform probe
+    // =========================================================================
+
+    static bool EndsWith(const char* s, const char* suffix) {
+        int sl = 0, xl = 0;
+        while (s[sl]) sl++;
+        while (suffix[xl]) xl++;
+        if (xl > sl) return false;
+        for (int i = 0; i < xl; i++) {
+            if (s[sl - xl + i] != suffix[i]) return false;
+        }
+        return true;
+    }
+
+    // The controller's UartSerialBusV2 descriptor (ACPI 6.4, 6.4.3.8.2.3)
+    // names the UART it hangs off in its ResourceSource string.
+    static bool ReadUartSource(int32_t node, char* source, int maxLength) {
+        Hal::AML::Object crs;
+        if (!Hal::AML::GetInterpreter().EvaluateChild(node, "_CRS", nullptr, 0, crs) ||
+            crs.Type != Hal::AML::ObjectType::Buffer) {
+            return false;
+        }
+        const uint8_t* data = crs.Buffer.Data;
+        uint32_t length = crs.Buffer.Length;
+        uint32_t pos = 0;
+        while (pos < length) {
+            uint8_t tag = data[pos];
+            if (!(tag & 0x80)) {
+                if ((tag & 0x78) == 0x78) break;          // end tag
+                pos += 1 + (tag & 0x07);
+                continue;
+            }
+            if (pos + 3 > length) break;
+            uint32_t descLength = (uint32_t)data[pos + 1] | ((uint32_t)data[pos + 2] << 8);
+            uint32_t end = pos + 3 + descLength;
+            if (end > length) break;
+            static constexpr uint8_t SerialBusTag = 0x8E, SerialBusUart = 3;
+            if (tag == SerialBusTag && descLength >= 9 && data[pos + 5] == SerialBusUart) {
+                uint32_t typeDataLength = (uint32_t)data[pos + 10] | ((uint32_t)data[pos + 11] << 8);
+                uint32_t s = pos + 12 + typeDataLength;
+                int n = 0;
+                while (s < end && data[s] != 0 && n < maxLength - 1) source[n++] = (char)data[s++];
+                source[n] = '\0';
+                return n > 0;
+            }
+            pos = end;
+        }
+        return false;
+    }
+
+    static bool ReadInteger(int32_t node, const char* name, uint64_t& value) {
+        Hal::AML::Object obj;
+        if (!Hal::AML::GetInterpreter().EvaluateChild(node, name, nullptr, 0, obj) ||
+            obj.Type != Hal::AML::ObjectType::Integer) {
+            return false;
+        }
+        value = obj.Integer;
+        return true;
+    }
+
+    bool ProbePlatform() {
+        Hal::AcpiDevices::DeviceInfo bt{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BCM2EA6", bt)) return false;
+
+        // Which UART: the descriptor's source, else the firmware's "UART in
+        // use" flag (0: the PL011 is the console, Bluetooth has the mini UART).
+        BtUart::Kind kind = BtUart::Kind::MiniUart;
+        char source[64] = {};
+        uint64_t uartInUse = 0;
+        if (ReadUartSource(bt.NodeIndex, source, sizeof(source))) {
+            kind = EndsWith(source, "URT0") ? BtUart::Kind::Pl011 : BtUart::Kind::MiniUart;
+        } else if (ReadInteger(bt.NodeIndex, "URIU", uartInUse)) {
+            kind = uartInUse == 0 ? BtUart::Kind::MiniUart : BtUart::Kind::Pl011;
+        }
+        bool mini = kind == BtUart::Kind::MiniUart;
+
+        Hal::AcpiDevices::DeviceInfo uart{};
+        if (!Hal::AcpiDevices::FindDeviceByHid(mini ? "BCM2836" : "BCM2837", uart)) {
+            KernelLogStream(WARNING, "BT-BCM") << bt.Path << ": its UART "
+                << (source[0] ? source : (mini ? "(mini UART)" : "(PL011)")) << " is not in ACPI";
+            return false;
+        }
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(uart.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            KernelLogStream(WARNING, "BT-BCM") << uart.Path << ": no register block in _CRS";
+            return false;
+        }
+
+        uint64_t console = Drivers::Serial::ConsoleAddress();
+        if (console != 0 && (console & ~0xFFFULL) == (base & ~0xFFFULL)) {
+            KernelLogStream(WARNING, "BT-BCM") << "Bluetooth's UART " << uart.Path
+                << " is the serial console; leaving Bluetooth off";
+            return false;
+        }
+
+        BtUart::Config config{};
+        config.kind = kind;
+        config.mmioPhys = base;
+        config.levelTriggered = true;
+        if (auto* irq = resources.FindFirst(Hal::AML::ResourceType::ExtendedIrq)) {
+            config.gsi = irq->ExtendedIrq.Interrupt;
+            config.levelTriggered = (irq->ExtendedIrq.Flags & 1) == 0;
+            config.activeLow = (irq->ExtendedIrq.Flags & 2) != 0;
+        }
+
+        // The mini UART's baud generator runs off the VPU core clock: ask the
+        // firmware for its current rate, else take the rate the firmware
+        // patched into the table (MUCR; the placeholder 0x1000000 means it
+        // did not). The PL011's reference clock is fixed (CLCK, 48 MHz).
+        uint64_t tableClock = 0;
+        uint32_t clockHz = 0;
+        const char* clockSource = "firmware";
+        if (mini) {
+            if (!RpiFirmware::GetClockRate(RpiFirmware::CLOCK_CORE, clockHz)) {
+                if (ReadInteger(uart.NodeIndex, "MUCR", tableClock) && tableClock > 0x1000000) {
+                    clockHz = (uint32_t)tableClock;
+                    clockSource = "ACPI MUCR";
+                } else {
+                    clockHz = 500000000;
+                    clockSource = "assumed";
+                }
+            }
+        } else {
+            clockSource = "PL011 reference";
+            clockHz = ReadInteger(uart.NodeIndex, "CLCK", tableClock) && tableClock != 0
+                ? (uint32_t)tableClock : 48000000;
+        }
+        config.clockHz = clockHz;
+
+        RpiGpio::Function function = mini ? RpiGpio::Function::Alt5 : RpiGpio::Function::Alt3;
+        bool pinsOk = RpiGpio::SetFunction(PIN_CTS, function)
+                   && RpiGpio::SetFunction(PIN_RTS, function)
+                   && RpiGpio::SetFunction(PIN_TXD, function)
+                   && RpiGpio::SetFunction(PIN_RXD, function)
+                   && RpiGpio::SetPull(PIN_CTS, RpiGpio::Pull::Up)
+                   && RpiGpio::SetPull(PIN_RTS, RpiGpio::Pull::None)
+                   && RpiGpio::SetPull(PIN_TXD, RpiGpio::Pull::None)
+                   && RpiGpio::SetPull(PIN_RXD, RpiGpio::Pull::Up);
+        if (!pinsOk) {
+            KernelLogStream(WARNING, "BT-BCM")
+                << "No GPIO controller (BCM2845): UART pins left as the firmware set them";
+        }
+
+        if (!BtUart::Initialize(config)) return false;
+
+        Hci::InitializeUart();
+        Hci::SetVendor(Hci::Vendor::Broadcom);
+        KernelLogStream(OK, "BT-BCM") << "Bluetooth controller " << bt.Path << " on "
+            << uart.Path << " (GSI " << (uint64_t)config.gsi << ", clock " << clockSource << ")";
+        return true;
+    }
+
+    // =========================================================================
+    // Controller bring-up
+    // =========================================================================
+
+    // BT_ON through the firmware's expander. The line has to be an output
+    // first (hci_bcm's GPIOD_OUT_LOW request does that on Linux); plain
+    // SET_GPIO_STATE is the fallback for firmware without the config tag.
+    static bool SetBtOn(bool on) {
+        return RpiFirmware::SetGpioOutput(RpiFirmware::EXPGPIO_BT_ON, on)
+            || RpiFirmware::SetGpioState(RpiFirmware::EXPGPIO_BT_ON, on);
+    }
+
+    // The controller samples its CTS (our RTS) as it leaves reset. Asserted,
+    // it boots into autobaud mode, where it answers HCI Reset and silently
+    // drops nearly everything else (on a Pi 400: Read Local Version, Read
+    // PIN Type, Write PIN Type, at any baud rate). hci_bcm deasserts it
+    // around power-on for the same reason; SyncAtInitialBaud asserts it
+    // again once the controller is up.
+    static void PowerCycle() {
+        if (g_powerSwitch) {
+            BtUart::DeassertRts();
+            if (SetBtOn(false)) {
+                Timekeeping::BusyWaitMs(20);
+                bool on = SetBtOn(true);
+                bool state = false;
+                bool readBack = RpiFirmware::GetGpioState(RpiFirmware::EXPGPIO_BT_ON, state);
+                KernelLogStream(on ? INFO : WARNING, "BT-BCM") << "BT_ON "
+                    << (on ? "switched on" : "did not switch on") << " (reads back "
+                    << (readBack ? (state ? "high" : "low") : "unknown") << ")";
+                // hci_bcm allows 100-120 ms to power on and leave reset.
+                Timekeeping::BusyWaitMs(150);
+            } else {
+                g_powerSwitch = false;
+                KernelLogStream(WARNING, "BT-BCM")
+                    << "Cannot switch BT_ON through the firmware; assuming the controller is powered";
+            }
+        }
+        Hci::ResetUartReceiver();
+    }
+
+    // Whether received bytes move by interrupt and whether any were lost:
+    // the first thing to know when the controller seems not to answer.
+    static void LogUartStats(const char* when) {
+        BtUart::Stats stats{};
+        BtUart::GetStats(stats);
+        KernelLogStream(INFO, "BT-BCM") << when << ": UART rx " << stats.rxBytes << " bytes, tx "
+            << stats.txBytes << ", " << stats.interrupts << " interrupts, " << stats.overruns
+            << " overruns, " << stats.rxDropped << " dropped, " << (uint64_t)BtUart::GetBaud()
+            << " baud, " << (BtUart::FlowControl() ? "RTS flow control" : "no flow control");
+    }
+
+    static bool ResetController(uint32_t timeoutMs) {
+        if (!Hci::SendCommand(Hci::OP_RESET, nullptr, 0)) return false;
+        if (!Hci::WaitCommandComplete(Hci::OP_RESET, nullptr, 0, timeoutMs)) return false;
+        // The controller acknowledges the reset before it has finished it
+        // and ignores commands until then; btbcm_reset waits 100 ms.
+        Timekeeping::BusyWaitMs(100);
+        return true;
+    }
+
+    // Reach the controller at its power-on 115200 baud and read its ROM
+    // version. A flow-control mode counts only once a second command has
+    // been answered too: a controller in autobaud mode answers HCI Reset
+    // alone, and a flow-control polarity that holds either side back fails
+    // here rather than later.
+    static bool SyncAtInitialBaud(Hci::LocalVersion& rom) {
+        BtUart::SetBaud(InitialBaud);
+        static constexpr bool attempts[] = { true, true, false, false };
+        for (bool flow : attempts) {
+            BtUart::SetFlowControl(flow);
+            // Events the controller held back while it saw RTS deasserted
+            // pour out once it is asserted: let them arrive and drop them,
+            // so they neither overrun the FIFO under the next reply nor
+            // leave the deframer mid-packet.
+            Timekeeping::BusyWaitMs(50);
+            Hci::ResetUartReceiver();
+            if (!ResetController(1000)) {
+                LogUartStats(flow ? "No reset reply with flow control" : "No reset reply");
+                continue;
+            }
+            // A controller slow to finish its reset gets a second chance.
+            bool answered = Hci::ReadLocalVersion(&rom);
+            if (!answered) {
+                Timekeeping::BusyWaitMs(200);
+                answered = Hci::ReadLocalVersion(&rom);
+            }
+            if (answered) {
+                g_flowControl = flow;
+                return true;
+            }
+            LogUartStats(flow ? "Reset answered but not Read Local Version, with flow control"
+                              : "Reset answered but not Read Local Version");
+        }
+        return false;
+    }
+
+    static uint32_t PickOperatingBaud() {
+        uint32_t wanted;
+        if (BtUart::GetKind() == BtUart::Kind::Pl011) {
+            wanted = g_flowControl ? 3000000 : 921600;
+        } else if (!BtUart::HasInterrupt()) {
+            // Polled 8-byte FIFO: stay where nothing overruns.
+            return InitialBaud;
+        } else {
+            wanted = g_flowControl ? 921600 : 460800;
+        }
+        static constexpr uint32_t rates[] = { 3000000, 2000000, 921600, 460800, 230400 };
+        for (uint32_t rate : rates) {
+            if (rate <= wanted && BtUart::BaudReachable(rate)) return rate;
+        }
+        return InitialBaud;
+    }
+
+    // Both ends to `baud`. The controller answers 0xFC18 at the old speed,
+    // then switches. False leaves the host at its old speed; the controller
+    // may or may not have switched (the caller power-cycles to be sure).
+    static bool SwitchBaud(uint32_t baud) {
+        uint32_t oldBaud = BtUart::GetBaud();
+        uint8_t params[6] = {
+            0x00, 0x00,
+            (uint8_t)baud, (uint8_t)(baud >> 8), (uint8_t)(baud >> 16), (uint8_t)(baud >> 24),
+        };
+        if (!Hci::SendCommand(Hci::OP_BCM_UPDATE_BAUD_RATE, params, sizeof(params)) ||
+            !Hci::WaitCommandComplete(Hci::OP_BCM_UPDATE_BAUD_RATE, nullptr, 0, 1000)) {
+            KernelLogStream(WARNING, "BT-BCM") << "Controller refused " << (uint64_t)baud << " baud";
+            return false;
+        }
+        if (!BtUart::SetBaud(baud)) return false;
+        Timekeeping::BusyWaitMs(10);
+        Hci::ResetUartReceiver();
+
+        Hci::LocalVersion ver{};
+        if (Hci::ReadLocalVersion(&ver)) return true;
+
+        KernelLogStream(WARNING, "BT-BCM") << "No answer at " << (uint64_t)baud << " baud";
+        BtUart::SetBaud(oldBaud);
+        Hci::ResetUartReceiver();
+        return false;
+    }
+
+    static const char* ChipFor(uint16_t subversion) {
+        for (const auto& chip : g_chips) {
+            if (chip.subversion == subversion) return chip.name;
+        }
+        return nullptr;
+    }
+
+    // Load 0:/os/firmware/brcm/<chip>.hcd. Returns nullptr (and logs) when
+    // the file is missing; the caller frees the image.
+    static uint8_t* LoadPatch(const char* chip, uint64_t& size) {
+        char path[64];
+        const char* prefix = "0:/os/firmware/brcm/";
+        int n = 0;
+        for (const char* p = prefix; *p; p++) path[n++] = *p;
+        for (const char* p = chip; *p && n < 56; p++) path[n++] = *p;
+        for (const char* p = ".hcd"; *p; p++) path[n++] = *p;
+        path[n] = '\0';
+
+        Fs::Vfs::BackendFile file;
+        if (Fs::Vfs::OpenBackendFile(path, file) < 0) {
+            KernelLogStream(WARNING, "BT-BCM") << "Firmware patch not found: " << path;
+            return nullptr;
+        }
+        size = Fs::Vfs::GetBackendFileSize(file);
+        uint8_t* image = size ? (uint8_t*)Memory::g_heap->Request(size) : nullptr;
+        if (image == nullptr) {
+            Fs::Vfs::CloseBackendFile(file);
+            return nullptr;
+        }
+        int got = Fs::Vfs::ReadBackendFile(file, image, 0, size);
+        Fs::Vfs::CloseBackendFile(file);
+        if (got != (int)size) {
+            Memory::g_heap->Free(image);
+            KernelLogStream(ERROR, "BT-BCM") << "Short read of " << path;
+            return nullptr;
+        }
+        KernelLogStream(INFO, "BT-BCM") << "Loading " << path << " (" << size << " bytes)";
+        return image;
+    }
+
+    static bool DownloadPatch(const uint8_t* image, uint64_t size) {
+        if (!Hci::SendCommand(Hci::OP_BCM_DOWNLOAD_MINIDRV, nullptr, 0) ||
+            !Hci::WaitCommandComplete(Hci::OP_BCM_DOWNLOAD_MINIDRV, nullptr, 0, 2000)) {
+            KernelLogStream(ERROR, "BT-BCM") << "Download Minidriver failed";
+            return false;
+        }
+        Timekeeping::BusyWaitMs(50);
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        uint64_t off = 0;
+        uint32_t records = 0;
+        while (off + 3 <= size) {
+            uint16_t opcode = (uint16_t)image[off] | ((uint16_t)image[off + 1] << 8);
+            uint8_t paramLength = image[off + 2];
+            off += 3;
+            if (off + paramLength > size) {
+                KernelLogStream(ERROR, "BT-BCM") << "Truncated patch record " << (uint64_t)records;
+                return false;
+            }
+            if (!Hci::SendCommand(opcode, image + off, paramLength)) return false;
+            bool acked = Hci::WaitCommandComplete(opcode, nullptr, 0, 2000);
+            if (!acked && opcode != Hci::OP_BCM_LAUNCH_RAM) {
+                KernelLogStream(ERROR, "BT-BCM") << "Patch record " << (uint64_t)records
+                    << " (opcode " << base::hex << (uint64_t)opcode << base::dec << ") failed";
+                return false;
+            }
+            off += paramLength;
+            records++;
+        }
+        // The patched firmware starts after Launch RAM; btbcm waits 250 ms.
+        Timekeeping::BusyWaitMs(250);
+        KernelLogStream(OK, "BT-BCM") << "Patch downloaded: " << (uint64_t)records << " records in "
+            << (Timekeeping::GetMilliseconds() - start) << " ms at "
+            << (uint64_t)BtUart::GetBaud() << " baud";
+        return true;
+    }
+
+    static void LogVersion(const char* chip, const Hci::LocalVersion& ver) {
+        uint16_t sub = ver.LmpSubversion;
+        KernelLogStream(OK, "BT-BCM") << (chip ? chip : "Broadcom controller") << " ("
+            << (uint64_t)((sub & 0xE000) >> 13) << "." << (uint64_t)((sub & 0x1F00) >> 8)
+            << "." << (uint64_t)(sub & 0x00FF) << ") build " << (uint64_t)(ver.HciRevision & 0x0FFF)
+            << ", " << (uint64_t)BtUart::GetBaud() << " baud, "
+            << (g_flowControl ? "RTS flow control" : "no flow control");
+    }
+
+    // One complete bring-up. `fast` raises the speed for the download and
+    // afterwards; without it everything stays at 115200.
+    static bool Attempt(bool fast) {
+        PowerCycle();
+        Hci::LocalVersion rom{};
+        if (!SyncAtInitialBaud(rom)) {
+            KernelLogStream(ERROR, "BT-BCM") << "Controller does not answer HCI commands"
+                << (g_powerSwitch ? "" : " (BT_ON was never switched on: it is likely unpowered)");
+            return false;
+        }
+        const char* chip = ChipFor(rom.LmpSubversion);
+        KernelLogStream(INFO, "BT-BCM") << "ROM firmware: LMP subversion " << base::hex
+            << (uint64_t)rom.LmpSubversion << ", HCI revision " << (uint64_t)rom.HciRevision
+            << base::dec << (chip ? " -> " : " (unknown chip)") << (chip ? chip : "");
+
+        uint32_t operating = fast ? PickOperatingBaud() : InitialBaud;
+        if (operating != InitialBaud && !SwitchBaud(operating)) return false;
+
+        uint64_t size = 0;
+        uint8_t* image = chip ? LoadPatch(chip, size) : nullptr;
+        if (image != nullptr) {
+            bool ok = DownloadPatch(image, size);
+            Memory::g_heap->Free(image);
+            if (!ok) return false;
+
+            // The patched firmware comes up at 115200 again.
+            BtUart::SetBaud(InitialBaud);
+            Hci::ResetUartReceiver();
+            bool reset = false;
+            for (int i = 0; i < 3 && !reset; i++) reset = ResetController(1000);
+            if (!reset) {
+                KernelLogStream(ERROR, "BT-BCM") << "Patched firmware does not answer HCI Reset";
+                return false;
+            }
+            if (operating != InitialBaud && !SwitchBaud(operating)) return false;
+        } else {
+            KernelLogStream(WARNING, "BT-BCM") << "Running the controller's ROM firmware";
+        }
+
+        Hci::LocalVersion ver{};
+        if (Hci::ReadLocalVersion(&ver)) LogVersion(chip, ver);
+        return true;
+    }
+
+    bool Initialize() {
+        if (Hci::GetTransport() != Hci::Transport::Uart) return false;
+        if (Attempt(true)) return true;
+        KernelLogStream(WARNING, "BT-BCM") << "Retrying bring-up at 115200 baud";
+        return Attempt(false);
+    }
+
+    // =========================================================================
+    // BD_ADDR
+    // =========================================================================
+
+    // Addresses are in HCI order (addr[0] = least significant octet).
+    static bool IsDefaultAddress(const uint8_t* addr) {
+        bool allZero = true, allOnes = true;
+        for (int i = 0; i < 6; i++) {
+            if (addr[i] != 0x00) allZero = false;
+            if (addr[i] != 0xFF) allOnes = false;
+        }
+        if (allZero || allOnes) return true;
+        // The chip-wide defaults Linux's btbcm lists for these parts (the
+        // BCM4345C5's is 43:45:C5:00:1F:AC) start with 0x43, which has the
+        // IEEE group bit set: no assigned device address does.
+        return (addr[5] & 0x01) != 0;
+    }
+
+    void FixDefaultAddress() {
+        uint8_t addr[6];
+        if (!Hci::ReadBdAddr(addr) || !IsDefaultAddress(addr)) return;
+
+        uint8_t mac[6];
+        if (!RpiFirmware::GetBoardMacAddress(mac)) {
+            KernelLogStream(WARNING, "BT-BCM")
+                << "Controller has its chip-wide default BD_ADDR and no board MAC is available";
+            return;
+        }
+        // The board's Ethernet MAC is unique to it and in the Raspberry Pi's
+        // own address block.
+        uint8_t board[6];
+        for (int i = 0; i < 6; i++) board[i] = mac[5 - i];
+        if (Hci::SetBdAddr(board)) {
+            KernelLogStream(OK, "BT-BCM") << "Replaced the chip-wide default BD_ADDR with the board's";
+        } else {
+            KernelLogStream(WARNING, "BT-BCM") << "Controller rejected the board BD_ADDR";
+        }
+    }
+
+}

kernel/src/Drivers/Bluetooth/Broadcom.hpp

@@ -0,0 +1,27 @@
+/*
+    * Broadcom.hpp
+    * Broadcom/Cypress UART Bluetooth controller bring-up (Raspberry Pi 4/400)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+
+namespace Drivers::Bluetooth::Broadcom {
+
+    // Find the on-board controller ACPI describes (BCM2EA6, \_SB.GDV0.BTH0),
+    // route its UART to GPIO 30-33 and start the UART and the H4 HCI
+    // transport. Needs no firmware files, so it runs during the boot driver
+    // probe. Returns false when there is no such controller.
+    bool ProbePlatform();
+
+    // Power-cycle the controller, load its firmware patch from
+    // 0:/os/firmware/brcm/ and raise the UART to its operating speed.
+    // Leaves the controller freshly reset and ready for standard HCI.
+    bool Initialize();
+
+    // After the last HCI Reset: give a controller still on its chip-wide
+    // default BD_ADDR (the CYW43456 ships with one) a board-unique address
+    // from the Pi's Ethernet MAC.
+    void FixDefaultAddress();
+
+}

kernel/src/Drivers/Bluetooth/Hci.cpp

@@ -1,6 +1,6 @@
 /*
     * Hci.cpp
-    * Bluetooth HCI transport over USB
+    * Bluetooth HCI transport over USB or a UART (H4)
     * Copyright (c) 2026 Daniel Hammer
 */
 
@@ -10,6 +10,7 @@
 #include "A2dp.hpp"
 #include <atomic>
 #include <Fs/Vfs.hpp>
+#include <Drivers/Serial/BtUart.hpp>
 #include <Drivers/USB/Xhci.hpp>
 #include <Drivers/USB/UsbDevice.hpp>
 #include <Terminal/Terminal.hpp>
@@ -22,12 +23,17 @@
 
 using namespace Kt;
 
-namespace Drivers::USB::Bluetooth::Hci {
+namespace Drivers::Bluetooth::Hci {
+
+    namespace Xhci = Drivers::USB::Xhci;
+    namespace BtUart = Drivers::Serial::BtUart;
 
     // =========================================================================
     // State
     // =========================================================================
 
+    static Transport g_transport = Transport::None;
+    static Vendor g_vendor = Vendor::Unknown;
     static uint8_t g_slotId = 0;
     static bool g_initialized = false;
 
@@ -541,6 +547,161 @@ namespace Drivers::USB::Bluetooth::Hci {
         }
     }
 
+    // =========================================================================
+    // UART (H4) receive deframing
+    // =========================================================================
+
+    // The UART delivers one byte stream: [type][header][payload]...  Pump()
+    // feeds it here in process context, so complete packets go straight to
+    // the same command mailbox / async event ring / ACL ring the USB
+    // callback fills.  A byte that cannot start a packet is dropped, which
+    // resynchronizes after line noise (e.g. while the controller powers up).
+    enum class H4State : uint8_t { Type, Header, Payload, Discard };
+
+    static H4State g_h4State = H4State::Type;
+    static uint8_t g_h4Type = 0;
+    static uint8_t g_h4Packet[ACL_RX_SLOT_SIZE] = {};
+    static uint16_t g_h4Len = 0;          // bytes in g_h4Packet
+    static uint16_t g_h4HeaderLen = 0;
+    static uint16_t g_h4Expected = 0;     // header + payload
+    static uint32_t g_h4Remaining = 0;    // Discard: bytes left to skip
+    static uint32_t g_h4Dropped = 0;      // bytes that could not start a packet
+
+    // The UART receive ring and the deframer have one consumer at a time:
+    // whoever holds this (Pump, or a flush).
+    static std::atomic<bool> g_uartConsumer{false};
+
+    void ResetUartReceiver() {
+        bool expected = false;
+        while (!g_uartConsumer.compare_exchange_weak(expected, true,
+                                                     std::memory_order_acquire)) {
+            expected = false;
+            Arch::Pause();
+        }
+        BtUart::FlushRx();
+        g_h4State = H4State::Type;
+        g_h4Len = 0;
+        g_h4Remaining = 0;
+        g_uartConsumer.store(false, std::memory_order_release);
+    }
+
+    static void CompleteH4Packet() {
+        if (g_h4Type == H4_EVENT) {
+            QueueCompleteHciEvent(g_h4Packet, g_h4Len);
+        } else if (g_h4Type == H4_ACL) {
+            QueueCompleteAclPacket(g_h4Packet, g_h4Len);
+        }
+    }
+
+    static void FeedH4Bytes(const uint8_t* data, uint32_t length) {
+        uint32_t off = 0;
+        while (off < length) {
+            switch (g_h4State) {
+                case H4State::Type: {
+                    uint8_t type = data[off++];
+                    g_h4Type = type;
+                    g_h4Len = 0;
+                    if (type == H4_EVENT)      g_h4HeaderLen = sizeof(EventHeader);
+                    else if (type == H4_ACL)   g_h4HeaderLen = sizeof(AclHeader);
+                    else if (type == H4_SCO)   g_h4HeaderLen = 3;
+                    else { g_h4Dropped++; break; }
+                    g_h4State = H4State::Header;
+                    break;
+                }
+
+                case H4State::Header: {
+                    g_h4Packet[g_h4Len++] = data[off++];
+                    if (g_h4Len < g_h4HeaderLen) break;
+                    uint32_t payload;
+                    if (g_h4Type == H4_EVENT)    payload = g_h4Packet[1];
+                    else if (g_h4Type == H4_ACL) payload = (uint32_t)g_h4Packet[2]
+                                                         | ((uint32_t)g_h4Packet[3] << 8);
+                    else                         payload = g_h4Packet[2];
+                    // SCO is not used; an ACL packet beyond the ring slot
+                    // cannot be a valid one from a controller that reported
+                    // its buffer size.  Skip either without losing framing.
+                    if (g_h4Type == H4_SCO
+                        || g_h4HeaderLen + payload > sizeof(g_h4Packet)) {
+                        g_h4Remaining = payload;
+                        g_h4State = payload ? H4State::Discard : H4State::Type;
+                        break;
+                    }
+                    g_h4Expected = (uint16_t)(g_h4HeaderLen + payload);
+                    if (payload == 0) {
+                        CompleteH4Packet();
+                        g_h4State = H4State::Type;
+                    } else {
+                        g_h4State = H4State::Payload;
+                    }
+                    break;
+                }
+
+                case H4State::Payload: {
+                    uint32_t take = g_h4Expected - g_h4Len;
+                    if (take > length - off) take = length - off;
+                    memcpy(&g_h4Packet[g_h4Len], &data[off], take);
+                    g_h4Len = (uint16_t)(g_h4Len + take);
+                    off += take;
+                    if (g_h4Len == g_h4Expected) {
+                        CompleteH4Packet();
+                        g_h4State = H4State::Type;
+                    }
+                    break;
+                }
+
+                case H4State::Discard: {
+                    uint32_t take = length - off;
+                    if (take > g_h4Remaining) take = g_h4Remaining;
+                    off += take;
+                    g_h4Remaining -= take;
+                    if (g_h4Remaining == 0) g_h4State = H4State::Type;
+                    break;
+                }
+            }
+        }
+    }
+
+    // =========================================================================
+    // Transport pump
+    // =========================================================================
+
+    void Pump() {
+        if (g_transport == Transport::Usb) {
+            Xhci::PollEvents();
+            return;
+        }
+        if (g_transport != Transport::Uart) return;
+
+        bool expected = false;
+        if (!g_uartConsumer.compare_exchange_strong(expected, true,
+                                                    std::memory_order_acquire)) return;
+        // Also moves bytes by hand, in case the UART interrupt never fires.
+        BtUart::Service();
+        uint8_t chunk[256];
+        int n;
+        while ((n = BtUart::Read(chunk, sizeof(chunk))) > 0) {
+            FeedH4Bytes(chunk, (uint32_t)n);
+        }
+        g_uartConsumer.store(false, std::memory_order_release);
+    }
+
+    bool InPumpContext() {
+        // UART packets are only queued by Pump, never handled inside it.
+        return g_transport == Transport::Usb && Xhci::InPollContext();
+    }
+
+    bool TransportHasRxData() {
+        return g_transport == Transport::Uart && BtUart::HasRxData();
+    }
+
+    Transport GetTransport() { return g_transport; }
+
+    uint32_t TransportBytesPerSecond() {
+        return g_transport == Transport::Uart ? BtUart::GetBaud() / 10 : 0;
+    }
+    void SetVendor(Vendor vendor) { g_vendor = vendor; }
+    Vendor GetVendor() { return g_vendor; }
+
     // =========================================================================
     // USB transfer callback
     // =========================================================================
@@ -641,7 +802,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     static void PollWait(uint32_t ms) {
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Timekeeping::GetMilliseconds() - start < ms) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
             for (int j = 0; j < 100; j++) {
                 Arch::CompilerBarrier();
@@ -672,10 +833,20 @@ namespace Drivers::USB::Bluetooth::Hci {
         // the configured USB function its settling delay.  StartEventPipe then
         // arms both receive endpoints before the first bootloader command.
 
+        g_transport = Transport::Usb;
         g_initialized = true;
         KernelLogStream(OK, "BT-HCI") << "HCI transport initialized on slot " << (uint64_t)slotId;
     }
 
+    void InitializeUart() {
+        // No DMA buffers: SendCommand/SendAcl copy each H4 packet straight
+        // into the UART's transmit ring.
+        ResetUartReceiver();
+        g_transport = Transport::Uart;
+        g_initialized = true;
+        KernelLogStream(OK, "BT-HCI") << "HCI transport initialized on UART";
+    }
+
     // Start receiving HCI events and ACL data — call after HCI init sequence.
     // Arms BOTH the interrupt IN (events) and the bulk IN (ACL).  The bulk IN
     // MUST stay armed across the firmware download: on this controller the device
@@ -712,7 +883,8 @@ namespace Drivers::USB::Bluetooth::Hci {
     // =========================================================================
 
     bool SendCommand(uint16_t opcode, const uint8_t* params, uint8_t paramLen) {
-        if (!g_initialized || !g_cmdDmaBuf) return false;
+        if (!g_initialized) return false;
+        if (g_transport == Transport::Usb && !g_cmdDmaBuf) return false;
 
         uint16_t expected = 0;
         if (!g_commandOpcode.compare_exchange_strong(expected, opcode,
@@ -745,6 +917,27 @@ namespace Drivers::USB::Bluetooth::Hci {
                 << base::hex << (uint64_t)opcode << base::dec;
         }
 
+        if (g_transport == Transport::Uart) {
+            // H4: [0x01][opcode][paramLen][params].  The transmit ring is
+            // shared with ACL traffic; wait briefly for room behind it.
+            uint8_t header[4] = {
+                H4_COMMAND, (uint8_t)(opcode & 0xFF), (uint8_t)(opcode >> 8), paramLen
+            };
+
+            uint64_t start = Timekeeping::GetMilliseconds();
+            while (!BtUart::Write(header, sizeof(header), params, paramLen)) {
+                if (Timekeeping::GetMilliseconds() - start > 1000) {
+                    FinishCommand(opcode);
+                    KernelLogStream(WARNING, "BT-HCI") << "SendCommand: UART transmit stalled, opcode="
+                        << base::hex << (uint64_t)opcode << base::dec;
+                    return false;
+                }
+                BtUart::Service();
+                Arch::Pause();
+            }
+            return true;
+        }
+
         // Use DMA-allocated buffer (not stack) for the command data.
         // xHCI reads from this buffer via DMA for OUT transfers.
         memset(g_cmdDmaBuf, 0, 512);
@@ -791,7 +984,7 @@ namespace Drivers::USB::Bluetooth::Hci {
         // cannot wait -- a nested PollEvents is a no-op, so the Command Complete
         // is reaped by the active PollEvents after we return.  The command was
         // already submitted (fire-and-forget); report success.
-        if (Xhci::InPollContext()) {
+        if (InPumpContext()) {
             FinishCommand(opcode);
             return true;
         }
@@ -799,7 +992,7 @@ namespace Drivers::USB::Bluetooth::Hci {
         uint64_t start = Timekeeping::GetMilliseconds();
 
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
 
             uint8_t event[sizeof(g_eventBuf)];
@@ -852,7 +1045,7 @@ namespace Drivers::USB::Bluetooth::Hci {
 
     bool WaitCommandStatus(uint16_t opcode, uint32_t timeoutMs) {
         // See WaitCommandComplete: cannot wait when nested under PollEvents.
-        if (Xhci::InPollContext()) {
+        if (InPumpContext()) {
             FinishCommand(opcode);
             return true;
         }
@@ -860,7 +1053,7 @@ namespace Drivers::USB::Bluetooth::Hci {
         uint64_t start = Timekeeping::GetMilliseconds();
 
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
 
             uint8_t event[sizeof(g_eventBuf)];
@@ -897,10 +1090,40 @@ namespace Drivers::USB::Bluetooth::Hci {
     // SendAcl — via USB bulk OUT
     // =========================================================================
 
+    // Room the UART transmit ring must keep for one maximum-size ACL packet.
+    static uint32_t UartAclSlotBytes() {
+        return 1u + sizeof(AclHeader) + (g_aclMaxLen ? g_aclMaxLen : 1021);
+    }
+
     bool SendAcl(uint16_t handle, uint16_t pbFlag, const uint8_t* data, uint16_t len) {
-        if (!g_initialized || !g_aclTxRing[0] || !GetConnection(handle)) return false;
+        if (!g_initialized || !GetConnection(handle)) return false;
         if (len + sizeof(AclHeader) > 4096) return false;  // Single page DMA buffer
 
+        if (g_transport == Transport::Uart) {
+            g_aclTxLock.Acquire();
+            if (g_aclMaxNum != 0 && AclPendingCount() >= g_aclMaxNum) {
+                g_aclTxLock.Release();
+                return false;
+            }
+            uint16_t handleFlags = (uint16_t)((handle & 0x0FFF) | pbFlag);
+            uint8_t header[1 + sizeof(AclHeader)] = {
+                H4_ACL,
+                (uint8_t)handleFlags, (uint8_t)(handleFlags >> 8),
+                (uint8_t)len, (uint8_t)(len >> 8)
+            };
+            // All or nothing: a partial H4 packet would desynchronize the
+            // controller's deframer.  A full ring drops the packet, as a
+            // full xHCI TX ring does.
+            bool queued = BtUart::Write(header, sizeof(header), data, len);
+            if (queued) {
+                g_aclPendingCount.fetch_add(1, std::memory_order_relaxed);
+                g_aclTxCount.fetch_add(1, std::memory_order_relaxed);
+            }
+            g_aclTxLock.Release();
+            return queued;
+        }
+
+        if (!g_aclTxRing[0]) return false;
         g_aclTxLock.Acquire();
 
         // HCI host flow control is a hard limit, not merely a media-pacing
@@ -951,6 +1174,11 @@ namespace Drivers::USB::Bluetooth::Hci {
     uint16_t AclMaxPackets()   { return g_aclMaxNum; }
 
     uint32_t AclTxInFlight() {
+        if (g_transport == Transport::Uart) {
+            uint32_t pending = BtUart::TxPending();
+            uint32_t slot = UartAclSlotBytes();
+            return (pending + slot - 1) / slot;
+        }
         uint32_t tx = g_aclTxCount.load(std::memory_order_relaxed);
         uint32_t done = g_aclTxDoneCount.load(std::memory_order_relaxed);
         return (tx >= done) ? (tx - done) : 0;
@@ -966,6 +1194,10 @@ namespace Drivers::USB::Bluetooth::Hci {
         uint16_t maxOut = g_aclMaxNum ? g_aclMaxNum : 4;
         uint16_t mediaLimit = maxOut > 1 ? (uint16_t)(maxOut - 1) : maxOut;
         if (AclPendingCount() >= mediaLimit) return false;
+        if (g_transport == Transport::Uart) {
+            // Room for this packet plus a signaling reply behind it.
+            return BtUart::TxFree() >= 2 * UartAclSlotBytes();
+        }
         if (AclTxInFlight() >= (uint32_t)(ACL_TX_SLOTS - 1)) return false;
         return true;
     }
@@ -1420,6 +1652,7 @@ namespace Drivers::USB::Bluetooth::Hci {
             }
 
             case EVT_VENDOR_SPECIFIC: {
+                if (g_vendor != Vendor::Intel) break;
                 // Intel vendor events carry a sub-opcode in the first byte.
                 //   0x02 = "bootup" (operational firmware booted after 0xFC01)
                 //   0x06 = "secure send result": result(1) opcode(2) status(1)
@@ -1625,7 +1858,7 @@ namespace Drivers::USB::Bluetooth::Hci {
         // copying past what the controller delivered.
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Timekeeping::GetMilliseconds() - start < 2000) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
 
             uint8_t event[sizeof(g_eventBuf)];
@@ -1676,7 +1909,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     void ProcessPendingCommands() {
         // Top-level only: drains queued pairing replies with real, confirmed
         // control transfers.  Must NOT be called from inside PollEvents.
-        if (Xhci::InPollContext()) return;
+        if (InPumpContext()) return;
         // One consumer at a time: this runs from the Connect wait loop AND the
         // idle-core service pump, potentially on different cores; two consumers
         // on the lockless queue would race the tail and interleave commands.
@@ -1728,7 +1961,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     bool WaitSecureSendResult(uint32_t timeoutMs, uint8_t* outResult, uint8_t* outStatus) {
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
             if (g_secureResultValid) {
                 if (outResult) *outResult = g_secureResult;
@@ -1763,7 +1996,7 @@ namespace Drivers::USB::Bluetooth::Hci {
                     << " byte " << g_ssBytesSent;
                 return false;
             }
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
 
             g_ssBytesSent += frag;
@@ -1784,7 +2017,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     // header is deliberately absent, and the NOCP credit counter is NOT
     // touched: these are commands, not ACL data).
     bool IntelSecureSendBulk(uint8_t fragmentType, const uint8_t* data, uint32_t len) {
-        if (!g_initialized || !g_aclTxRing[0]) return false;
+        if (!g_initialized || g_transport != Transport::Usb || !g_aclTxRing[0]) return false;
         if (g_aclTxTransportError.load(std::memory_order_acquire)) return false;
 
         uint32_t off = 0;
@@ -1796,7 +2029,7 @@ namespace Drivers::USB::Bluetooth::Hci {
             // controller from freezing the boot path.
             uint64_t start = Timekeeping::GetMilliseconds();
             while (AclTxInFlight() >= (uint32_t)(ACL_TX_SLOTS - 1)) {
-                Xhci::PollEvents();
+                Pump();
                 DrainEvents();
                 if (g_aclTxTransportError.load(std::memory_order_acquire)) {
                     KernelLogStream(ERROR, "BT-HCI")
@@ -1843,7 +2076,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     bool DrainBulkTx(uint32_t timeoutMs) {
         uint64_t start = Timekeeping::GetMilliseconds();
         while (AclTxInFlight() > 0) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
             if (g_aclTxTransportError.load(std::memory_order_acquire)) return false;
             if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
@@ -1872,7 +2105,7 @@ namespace Drivers::USB::Bluetooth::Hci {
 
         uint64_t start = Timekeeping::GetMilliseconds();
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Pump();
             DrainEvents();
             if (g_intelBootup) return true;
             for (int j = 0; j < 100; j++) Arch::CompilerBarrier();
@@ -1953,9 +2186,14 @@ namespace Drivers::USB::Bluetooth::Hci {
     }
 
     bool SetBdAddr(const uint8_t* addr) {
-        // Intel 0xFC31: 6-byte BD_ADDR, little-endian (same order as ReadBdAddr).
-        if (!SendCommand(OP_INTEL_WRITE_BD_ADDR, addr, 6)) return false;
-        return WaitCommandComplete(OP_INTEL_WRITE_BD_ADDR, nullptr, 0, 2000);
+        // Intel 0xFC31 / Broadcom 0xFC01: 6-byte BD_ADDR, little-endian (same
+        // order as ReadBdAddr).
+        uint16_t opcode;
+        if (g_vendor == Vendor::Intel) opcode = OP_INTEL_WRITE_BD_ADDR;
+        else if (g_vendor == Vendor::Broadcom) opcode = OP_BCM_WRITE_BD_ADDR;
+        else return false;
+        if (!SendCommand(opcode, addr, 6)) return false;
+        return WaitCommandComplete(opcode, nullptr, 0, 2000);
     }
 
     bool Disconnect(uint16_t handle, uint8_t reason) {

kernel/src/Drivers/Bluetooth/Hci.hpp

@@ -1,23 +1,35 @@
 /*
     * Hci.hpp
     * Bluetooth HCI (Host Controller Interface) layer
-    * HCI transport over USB bulk/interrupt/control endpoints
+    * HCI transport over USB (bulk/interrupt/control endpoints) or a UART (H4)
     * Copyright (c) 2026 Daniel Hammer
 */
 
 #pragma once
 #include <cstdint>
 
-namespace Drivers::USB::Bluetooth::Hci {
+namespace Drivers::Bluetooth::Hci {
 
     // =========================================================================
-    // HCI packet types (for USB transport)
+    // HCI transports
     // =========================================================================
 
     // USB transport uses different endpoints for each packet type:
     // Commands  -> Control EP0 (class request)
     // ACL data  -> Bulk OUT / Bulk IN
     // Events    -> Interrupt IN
+    //
+    // The UART transport (H4, Bluetooth Core spec Vol 4 Part A) carries all
+    // of them on one byte stream, each packet preceded by its type.
+    constexpr uint8_t H4_COMMAND = 0x01;
+    constexpr uint8_t H4_ACL     = 0x02;
+    constexpr uint8_t H4_SCO     = 0x03;
+    constexpr uint8_t H4_EVENT   = 0x04;
+
+    enum class Transport : uint8_t { None, Usb, Uart };
+
+    // Controller vendor, for the vendor-specific commands (BD_ADDR write).
+    enum class Vendor : uint8_t { Unknown, Intel, Broadcom };
 
     // =========================================================================
     // HCI command opcodes (OGF << 10 | OCF)
@@ -61,6 +73,13 @@ namespace Drivers::USB::Bluetooth::Hci {
     constexpr uint16_t OP_READ_LOCAL_FEATURES    = 0x1003;
     constexpr uint16_t OP_READ_BUFFER_SIZE       = 0x1005;
 
+    // Broadcom vendor commands (OGF 0x3F), from Linux btbcm/hci_bcm
+    constexpr uint16_t OP_BCM_WRITE_BD_ADDR      = 0xFC01;
+    constexpr uint16_t OP_BCM_UPDATE_BAUD_RATE   = 0xFC18;
+    constexpr uint16_t OP_BCM_DOWNLOAD_MINIDRV   = 0xFC2E;
+    constexpr uint16_t OP_BCM_WRITE_RAM          = 0xFC4C;
+    constexpr uint16_t OP_BCM_LAUNCH_RAM         = 0xFC4E;
+
     // Intel vendor commands (OGF 0x3F)
     constexpr uint16_t OP_INTEL_READ_VERSION     = 0xFC05;
     constexpr uint16_t OP_INTEL_RESET            = 0xFC01;
@@ -175,6 +194,37 @@ namespace Drivers::USB::Bluetooth::Hci {
     // Initialize HCI transport over USB for the given slot
     void Initialize(uint8_t slotId);
 
+    // Initialize HCI transport over the Bluetooth UART (Serial::BtUart,
+    // already initialized by the caller).
+    void InitializeUart();
+
+    Transport GetTransport();
+
+    // Bytes per second the host-to-controller link can carry: the UART's
+    // line rate (8N1), or 0 when the transport is not the bottleneck (USB).
+    uint32_t TransportBytesPerSecond();
+
+    void SetVendor(Vendor vendor);
+    Vendor GetVendor();
+
+    // Move received bytes from the transport into the HCI queues: the xHCI
+    // event ring for USB, the UART receive ring (H4 deframing) for UART.
+    // Every wait loop calls this before DrainEvents.  Non-reentrant; a
+    // nested or concurrent call returns at once.
+    void Pump();
+
+    // True while running nested under Pump (USB transfer callbacks), where
+    // a command cannot be waited for.
+    bool InPumpContext();
+
+    // Received transport bytes are waiting for Pump (UART only; USB signals
+    // through the xHCI deferred-work flag).
+    bool TransportHasRxData();
+
+    // Discard all received UART input, queued bytes and any partial packet
+    // (after a power cycle or a baud-rate change).
+    void ResetUartReceiver();
+
     // Start receiving HCI events and ACL data (call after HCI init sequence).
     // Arms both the interrupt IN and the bulk IN; the bulk IN must stay armed
     // through the firmware download (it absorbs the device's ~635 KB cc=4 glitch
@@ -186,7 +236,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     // are intentionally ignored and any endpoint error is recovered here.
     void EnableAclDataReception();
 
-    // Send an HCI command via USB control transfer (EP0)
+    // Send an HCI command (USB: control transfer on EP0; UART: H4 packet)
     bool SendCommand(uint16_t opcode, const uint8_t* params, uint8_t paramLen);
 
     // Wait for a Command Complete event matching the given opcode
@@ -197,7 +247,7 @@ namespace Drivers::USB::Bluetooth::Hci {
     // Wait for a Command Status event matching the given opcode
     bool WaitCommandStatus(uint16_t opcode, uint32_t timeoutMs = 2000);
 
-    // Send ACL data via USB bulk OUT
+    // Send ACL data (USB: bulk OUT; UART: H4 packet)
     bool SendAcl(uint16_t handle, uint16_t pbFlag, const uint8_t* data, uint16_t len);
 
     // Process an HCI event received on the interrupt IN endpoint
@@ -323,8 +373,10 @@ namespace Drivers::USB::Bluetooth::Hci {
     // Drives Link Key Request -> pairing; needed for bonded-device reconnects.
     bool AuthenticateLink(uint16_t handle);
 
-    // Set the adapter's BD_ADDR (Intel vendor command 0xFC31).  Used to dodge a
-    // remote that holds a stale, un-clearable bond to our real address.
+    // Set the adapter's BD_ADDR (vendor command: Intel 0xFC31, Broadcom
+    // 0xFC01).  Used to dodge a remote that holds a stale, un-clearable bond
+    // to our real address, and to give Broadcom parts without a programmed
+    // address a unique one.
     bool SetBdAddr(const uint8_t* addr);
 
     // Send any queued pairing replies (IO-cap / user-confirm / link-key) with
@@ -340,7 +392,8 @@ namespace Drivers::USB::Bluetooth::Hci {
     uint16_t AclMaxPackets();
 
     // ACL packets handed to the xHCI whose bulk OUT completion has not been
-    // reaped yet (each one still owns a TX DMA ring slot).
+    // reaped yet (each one still owns a TX DMA ring slot).  On UART, the
+    // transmit ring's backlog in maximum-size packets (0 once drained).
     uint32_t AclTxInFlight();
 
     // True when SendAcl can be called without overrunning either the

kernel/src/Drivers/Bluetooth/IntelFirmware.cpp

@@ -23,7 +23,7 @@
 
 using namespace Kt;
 
-namespace Drivers::USB::Bluetooth {
+namespace Drivers::Bluetooth {
 
     // =========================================================================
     // .sfi / TLV format constants (from drivers/bluetooth/btintel.{c,h})

kernel/src/Drivers/Bluetooth/IntelFirmware.hpp

@@ -7,7 +7,7 @@
 #pragma once
 #include <cstdint>
 
-namespace Drivers::USB::Bluetooth {
+namespace Drivers::Bluetooth {
 
     // Drive the Intel bootloader firmware download sequence for the currently
     // registered adapter:

kernel/src/Drivers/Bluetooth/L2cap.cpp

@@ -9,7 +9,6 @@
 #include "Hci.hpp"
 #include "A2dp.hpp"
 #include "Avrcp.hpp"
-#include <Drivers/USB/Xhci.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 #include <Libraries/Memory.hpp>
@@ -17,7 +16,7 @@
 
 using namespace Kt;
 
-namespace Drivers::USB::Bluetooth::L2cap {
+namespace Drivers::Bluetooth::L2cap {
 
     // =========================================================================
     // State
@@ -548,7 +547,7 @@ namespace Drivers::USB::Bluetooth::L2cap {
         uint64_t start = Timekeeping::GetMilliseconds();
 
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            Xhci::PollEvents();
+            Hci::Pump();
             Hci::DrainEvents();   // process incoming ACL (the Connection Response!)
 
             auto* ch = GetChannel(localCid);

kernel/src/Drivers/Bluetooth/L2cap.hpp

@@ -7,7 +7,7 @@
 #pragma once
 #include <cstdint>
 
-namespace Drivers::USB::Bluetooth::L2cap {
+namespace Drivers::Bluetooth::L2cap {
 
     // =========================================================================
     // L2CAP CIDs (Channel Identifiers)

kernel/src/Drivers/Bluetooth/Sbc.cpp

@@ -9,7 +9,7 @@
 #include "Sbc.hpp"
 #include <Libraries/Memory.hpp>
 
-namespace Drivers::USB::Bluetooth::Sbc {
+namespace Drivers::Bluetooth::Sbc {
 
     // =========================================================================
     // Analysis filterbank tables

kernel/src/Drivers/Bluetooth/Sbc.hpp

@@ -8,7 +8,7 @@
 #pragma once
 #include <cstdint>
 
-namespace Drivers::USB::Bluetooth::Sbc {
+namespace Drivers::Bluetooth::Sbc {
 
     // =========================================================================
     // SBC configuration

kernel/src/Drivers/Init.cpp

@@ -10,15 +10,22 @@
 #include <Drivers/Graphics/DgpuPower.hpp>
 #include <Drivers/Net/E1000.hpp>
 #include <Drivers/Net/E1000E.hpp>
+#include <Drivers/Net/Genet.hpp>
 #include <Drivers/Net/Wifi/Wifi.hpp>
 #include <Drivers/USB/Xhci.hpp>
 #include <Drivers/Storage/Ahci.hpp>
 #include <Drivers/Storage/Nvme.hpp>
+#include <Drivers/Storage/Sdhci.hpp>
 #include <Drivers/Storage/Gpt.hpp>
 #include <Drivers/Audio/IntelHda.hpp>
+#include <Drivers/Bluetooth/Bluetooth.hpp>
 #include <Graphics/Framebuffer.hpp>
 #include <Net/Net.hpp>
 #include <Terminal/Terminal.hpp>
+#include <ACPI/AcpiDevices.hpp>
+#include <Boot/Boot.hpp>
+#include <ACPI/AML/AmlInterpreter.hpp>
+#include <ACPI/AML/AmlResource.hpp>
 
 namespace Drivers {
 
@@ -89,6 +96,10 @@ namespace Drivers {
         return Storage::Nvme::Probe(dev);
     }
 
+    static bool ProbeSdhci(const Pci::PciDevice& dev) {
+        return Storage::Sdhci::Probe(dev);
+    }
+
     static bool ProbeIntelHda(const Pci::PciDevice& dev) {
         return Audio::IntelHda::Probe(dev);
     }
@@ -189,6 +200,18 @@ namespace Drivers {
         //          also reports 0x01, but it exposes an I/O-space BAR0:
         //          ReadBar0 returns 0 for I/O BARs and Probe rejects it
         //          cleanly, so matching it here is harmless.
+        // SD host controller (standard interface) - class 0x08/0x05
+        {
+            "SDHCI",
+            0,
+            0x08,                           // ClassCode (Base system peripheral)
+            0x05,                           // SubClass (SD host controller)
+            0xFF,
+            nullptr,
+            0,
+            Pci::ProbePhase::Normal,
+            ProbeSdhci,
+        },
         {
             "IntelHDA",
             0x8086,                         // VendorId (Intel)
@@ -227,8 +250,243 @@ namespace Drivers {
         Graphics::DgpuPower::PowerDownUnused();
     }
 
+    // A device's _DMA (or its parent's) bounds the addresses it can reach.
+    // DMA buffers come from anywhere in RAM, so RAM past the bound is a
+    // configuration error: on the Raspberry Pi 4 the firmware's "Limit RAM
+    // to 3 GB" setting must stay on.
+    static void WarnIfRamBeyondDma(const char* devicePath) {
+        char path[160];
+        int len = 0;
+        while (devicePath[len] != '\0' && len < 120) { path[len] = devicePath[len]; len++; }
+
+        Hal::AML::ResourceList dma;
+        bool found = false;
+        for (int attempt = 0; attempt < 2 && !found; attempt++) {
+            const char suffix[] = "._DMA";
+            for (int i = 0; i < (int)sizeof(suffix); i++) path[len + i] = suffix[i];
+            Hal::AML::Object obj;
+            if (Hal::AML::GetInterpreter().EvaluateObject(path, obj) &&
+                obj.Type == Hal::AML::ObjectType::Buffer &&
+                Hal::AML::ParseResourceTemplate(obj.Buffer.Data, obj.Buffer.Length, dma)) {
+                found = true;
+                break;
+            }
+            // Retry on the parent device.
+            while (len > 0 && path[len - 1] != '.') len--;
+            if (len == 0) return;
+            len--;
+        }
+
+        uint64_t dmaBase = 0, dmaLength = 0;
+        if (!found || !dma.GetMemory(dmaBase, dmaLength) || dmaLength == 0) return;
+        uint64_t dmaEnd = dmaBase + dmaLength;
+
+        const auto& map = montauk::boot::Info().memoryMap;
+        for (uint64_t i = 0; i < map.count; i++) {
+            const auto& r = map.regions[i];
+            if (r.kind == montauk::boot::MemoryKind::Usable && r.base + r.length > dmaEnd) {
+                Kt::KernelLogStream(Kt::WARNING, "Drivers") << devicePath << " can only reach RAM below 0x"
+                    << base::hex << dmaEnd << ", but there is RAM up to 0x" << (r.base + r.length)
+                    << ": its DMA may fail (Raspberry Pi 4: keep 'Limit RAM to 3 GB' enabled)";
+                return;
+            }
+        }
+    }
+
+    // An xHCI controller ACPI describes as a platform device (PNP0D10)
+    // instead of a PCI function: the Raspberry Pi 4/400's VL805 under UEFI
+    // firmware in ACPI mode, where firmware has set up the non-ECAM PCIe
+    // behind it. Its DMA is non-coherent unless _CCA says otherwise.
+    static void ProbePlatformXhci() {
+        if (USB::Xhci::IsInitialized()) return;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("PNP0D10", info) &&
+            !Hal::AcpiDevices::FindDeviceByHid("PNP0D15", info)) {
+            return;
+        }
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            Kt::KernelLogStream(Kt::WARNING, "Drivers") << "xHCI " << info.Path
+                << ": no register block in _CRS";
+            return;
+        }
+
+        uint32_t gsi = 0;
+        bool level = true, activeLow = false;
+        if (auto* irq = resources.FindFirst(Hal::AML::ResourceType::ExtendedIrq)) {
+            gsi = irq->ExtendedIrq.Interrupt;
+            level = (irq->ExtendedIrq.Flags & 1) == 0;
+            activeLow = (irq->ExtendedIrq.Flags & 2) != 0;
+        }
+
+        bool coherent = false;
+        Hal::AML::Object cca;
+        if (Hal::AML::GetInterpreter().EvaluateChild(info.NodeIndex, "_CCA", nullptr, 0, cca) &&
+            cca.Type == Hal::AML::ObjectType::Integer) {
+            coherent = cca.Integer != 0;
+        }
+
+        Kt::KernelLogStream(Kt::INFO, "Drivers") << "xHCI platform device " << info.Path
+            << " at 0x" << base::hex << base << ", GSI " << base::dec << (uint64_t)gsi
+            << (coherent ? ", coherent DMA" : ", non-coherent DMA");
+        WarnIfRamBeyondDma(info.Path);
+        USB::Xhci::ProbePlatform(base, length, gsi, level, activeLow, coherent);
+    }
+
+    // SD host controllers ACPI describes directly: the Raspberry Pi 4's
+    // EMMC2 (BRCME88C, the SD card slot) or a standard PNP0D40.
+    static void ProbePlatformSdhci() {
+        if (Storage::Sdhci::IsInitialized()) return;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BRCME88C", info) &&
+            !Hal::AcpiDevices::FindDeviceByHid("PNP0D40", info)) {
+            return;
+        }
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            return;
+        }
+        Storage::Sdhci::ProbePlatform(base, info.Path);
+    }
+
+    // A device property (_DSD: { UUID, Package { Package { key, value },
+    // ... } }) by key.
+    static bool FindDsdProperty(uint32_t nodeIndex, const char* key, Hal::AML::Object& value) {
+        auto& interp = Hal::AML::GetInterpreter();
+        Hal::AML::Object dsd, properties;
+        if (!interp.EvaluateChild(nodeIndex, "_DSD", nullptr, 0, dsd) ||
+            dsd.Type != Hal::AML::ObjectType::Package ||
+            !interp.GetPackageElement(dsd, 1, properties) ||
+            properties.Type != Hal::AML::ObjectType::Package) {
+            return false;
+        }
+
+        int count = interp.GetPackageCount(properties);
+        for (int i = 0; i < count; i++) {
+            Hal::AML::Object pair, name;
+            if (!interp.GetPackageElement(properties, i, pair) || pair.Type != Hal::AML::ObjectType::Package ||
+                !interp.GetPackageElement(pair, 0, name) || name.Type != Hal::AML::ObjectType::String) {
+                continue;
+            }
+            int c = 0;
+            while (key[c] != '\0' && name.String.Data[c] == key[c]) c++;
+            if (key[c] != '\0' || name.String.Data[c] != '\0') continue;
+            return interp.GetPackageElement(pair, 1, value);
+        }
+        return false;
+    }
+
+    static bool StringEquals(const char* a, const char* b) {
+        while (*a != '\0' && *a == *b) { a++; b++; }
+        return *a == *b;
+    }
+
+    // The Raspberry Pi 4/400's GENET v5 Ethernet MAC (BCM6E4E). The first
+    // of its two interrupts carries the default queue.
+    static void ProbePlatformGenet() {
+        if (Net::Genet::IsInitialized()) return;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BCM6E4E", info)) return;
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            Kt::KernelLogStream(Kt::WARNING, "Drivers") << "GENET " << info.Path
+                << ": no register block in _CRS";
+            return;
+        }
+
+        uint32_t gsi = 0;
+        bool level = true, activeLow = false;
+        if (auto* irq = resources.FindFirst(Hal::AML::ResourceType::ExtendedIrq)) {
+            gsi = irq->ExtendedIrq.Interrupt;
+            level = (irq->ExtendedIrq.Flags & 1) == 0;
+            activeLow = (irq->ExtendedIrq.Flags & 2) != 0;
+        }
+
+        bool coherent = false;
+        Hal::AML::Object cca;
+        if (Hal::AML::GetInterpreter().EvaluateChild(info.NodeIndex, "_CCA", nullptr, 0, cca) &&
+            cca.Type == Hal::AML::ObjectType::Integer) {
+            coherent = cca.Integer != 0;
+        }
+
+        // The Pi 4's PHY delays the receive clock and expects the MAC to
+        // delay transmit: "rgmii-rxid", also when the table says nothing.
+        auto phyMode = Net::Genet::PhyMode::RgmiiRxid;
+        Hal::AML::Object mode;
+        if (FindDsdProperty(info.NodeIndex, "phy-mode", mode) && mode.Type == Hal::AML::ObjectType::String) {
+            const char* m = mode.String.Data;
+            if (StringEquals(m, "rgmii")) phyMode = Net::Genet::PhyMode::Rgmii;
+            else if (StringEquals(m, "rgmii-id")) phyMode = Net::Genet::PhyMode::RgmiiId;
+            else if (StringEquals(m, "rgmii-txid")) phyMode = Net::Genet::PhyMode::RgmiiTxid;
+            else if (!StringEquals(m, "rgmii-rxid")) {
+                Kt::KernelLogStream(Kt::WARNING, "Drivers") << "GENET " << info.Path
+                    << ": unsupported phy-mode " << m << ", trying rgmii-rxid";
+            }
+        }
+
+        uint32_t burst = 0;
+        Hal::AML::Object burstObj;
+        if (FindDsdProperty(info.NodeIndex, "brcm,max-dma-burst-size", burstObj) &&
+            burstObj.Type == Hal::AML::ObjectType::Integer) {
+            burst = (uint32_t)burstObj.Integer;
+        }
+
+        Kt::KernelLogStream(Kt::INFO, "Drivers") << "GENET platform device " << info.Path
+            << " at 0x" << base::hex << base << ", GSI " << base::dec << (uint64_t)gsi;
+        WarnIfRamBeyondDma(info.Path);
+        Net::Genet::ProbePlatform(base, length, gsi, level, activeLow, coherent, phyMode, burst);
+    }
+
+    // The Raspberry Pi 4/400's Wi-Fi: the CYW4345x's SDIO half on the
+    // BCM2711's older Arasan controller (BCM2847, GPIO 34-39). Only while the
+    // card slot is on EMMC2 (BRCME88C): with the firmware's "SD on Arasan"
+    // option the Arasan runs the slot instead, and is not the radio's.
+    static void ProbePlatformBroadcomWifi() {
+        Hal::AcpiDevices::DeviceInfo emmc2{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BRCME88C", emmc2)) return;
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        Hal::AcpiDevices::DeviceInfo info{};
+        const char* name = "BCM2847";
+        if (Hal::AcpiDevices::FindDeviceByHid("BCM2847", info) &&
+            Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) &&
+            resources.GetMemory(base, length) && base != 0) {
+            name = info.Path;
+        } else {
+            // Not described: the Arasan sits 256 KiB below EMMC2 on the
+            // BCM2711 (0x7E300000 / 0x7E340000 bus addresses).
+            Hal::AML::ResourceList emmc2Resources;
+            uint64_t emmc2Base = 0;
+            if (!Hal::AcpiDevices::EvaluateCrs(emmc2.NodeIndex, emmc2Resources) ||
+                !emmc2Resources.GetMemory(emmc2Base, length) || emmc2Base < 0x40000) {
+                return;
+            }
+            base = emmc2Base - 0x40000;
+            name = "the Arasan controller (inferred from EMMC2)";
+        }
+        Net::Wifi::ProbeBroadcomSdio(base, name);
+    }
+
     void ProbeNormal() {
         Pci::ProbeAll(g_driverTable, g_driverTableCount, Pci::ProbePhase::Normal);
+        ProbePlatformXhci();
+        ProbePlatformSdhci();
+        ProbePlatformGenet();
+        ProbePlatformBroadcomWifi();
+        Bluetooth::ProbePlatformAdapter();
     }
 
     void InitializeNetwork() {
@@ -241,10 +499,15 @@ namespace Drivers {
         Storage::Gpt::ProbeAll();
     }
 
+    void ServiceStorageHotplug() {
+        if (USB::Xhci::HasDeferredWork()) USB::Xhci::ProcessDeferredWork();
+    }
+
     void InitializeAudio() {
         // HDA driver initializes during ProbeNormal().
-        // Bluetooth audio (A2DP) initializes automatically during USB enumeration
-        // when a Bluetooth adapter is detected by the xHCI driver.
+        // Bluetooth audio (A2DP) initializes with its adapter: during USB
+        // enumeration, or for an on-board UART controller (Raspberry Pi) from
+        // the platform probe in ProbeNormal().
     }
 
 }

kernel/src/Drivers/Init.hpp

@@ -29,6 +29,11 @@ namespace Drivers {
     // Post-probe: register SATA drives with VFS.
     void InitializeStorage();
 
+    // Before the scheduler runs: enumerate storage that arrived after the boot
+    // scan (a USB device still training, a hub port), which normally waits for
+    // the idle loop. Its partitions are mounted as it registers.
+    void ServiceStorageHotplug();
+
     // Post-probe: initialize audio subsystem.
     void InitializeAudio();
 

kernel/src/Drivers/Net/E1000.cpp

@@ -14,6 +14,7 @@
 #include <Libraries/Memory.hpp>
 #include <Arch/Interrupts.hpp>
 #include <CppLib/Spinlock.hpp>
+#include <Arch/Cpu.hpp>
 #include <atomic>
 
 using namespace Kt;
@@ -491,6 +492,7 @@ namespace Drivers::Net::E1000 {
 
         // Advance the tail pointer (tells the NIC there's a new packet)
         g_txTail = (g_txTail + 1) % TX_DESC_COUNT;
+        Arch::DmaWriteFence();
         WriteReg(REG_TDT, g_txTail);
 
         g_txPacketCount++;
@@ -545,6 +547,7 @@ namespace Drivers::Net::E1000 {
             uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
             RxDescriptor& desc = g_rxDescs[nextIdx];
             if (!(desc.Status & RXSTA_DD)) break;
+            Arch::DmaReadFence();
 
             uint16_t length = desc.Length;
             g_rxPacketCount++;
@@ -554,6 +557,7 @@ namespace Drivers::Net::E1000 {
             desc.Length = 0;
             desc.Errors = 0;
             g_rxTail = nextIdx;
+            Arch::DmaWriteFence();
             WriteReg(REG_RDT, g_rxTail);
             processed++;
         }

kernel/src/Drivers/Net/E1000E.cpp

@@ -667,6 +667,7 @@ namespace Drivers::Net::E1000E {
             if (!(desc.Status & RXSTA_DD)) {
                 break;
             }
+            Arch::DmaReadFence();
 
             uint16_t length = desc.Length;
             g_rxPacketCount++;
@@ -680,6 +681,7 @@ namespace Drivers::Net::E1000E {
             desc.Errors = 0;
 
             g_rxTail = nextIdx;
+            Arch::DmaWriteFence();
             WriteReg(REG_RDT, g_rxTail);
             processed++;
         }
@@ -709,6 +711,7 @@ namespace Drivers::Net::E1000E {
         desc.Status = 0;
 
         g_txTail = (g_txTail + 1) % TX_DESC_COUNT;
+        Arch::DmaWriteFence();
         WriteReg(REG_TDT, g_txTail);
 
         g_txPacketCount++;

kernel/src/Drivers/Net/Genet.cpp

@@ -0,0 +1,713 @@
+/*
+    * Genet.cpp
+    * Broadcom GENET v5 Ethernet driver (Raspberry Pi 4 / 400)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Genet.hpp"
+#include <Arch/Cpu.hpp>
+#include <Arch/Interrupts.hpp>
+#include <ACPI/GsiRouting.hpp>
+#include <Drivers/Platform/RpiFirmware.hpp>
+#include <Memory/Dma.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Memory/PageFrameAllocator.hpp>
+#include <Libraries/Memory.hpp>
+#include <Timekeeping/Timer.hpp>
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
+#include <CppLib/Spinlock.hpp>
+#include <atomic>
+
+using namespace Kt;
+
+namespace Drivers::Net::Genet {
+
+    // ====================================================================
+    // State
+    // ====================================================================
+
+    static bool g_initialized = false;
+    static bool g_coherent = false;
+    static PhyMode g_phyMode = PhyMode::RgmiiRxid;
+    static volatile uint8_t* g_regs = nullptr;
+    static uint8_t g_mac[6] = {};
+    static uint32_t g_phyAddr = 1;
+
+    // Packet buffers, BUF_LENGTH each, two to a page. Slot n of a ring
+    // always uses buffer n.
+    static uint8_t* g_rxBufs[RX_DESCS] = {};
+    static uint8_t* g_txBufs[TX_DESCS] = {};
+
+    // Software copies of the 16-bit ring counters: the RX consumer index we
+    // last wrote and the TX producer index. Slot = counter % ring size.
+    static std::atomic<uint32_t> g_rxCons{0};
+    static uint32_t g_txProd = 0;
+    static kcp::Spinlock g_txLock;
+
+    static std::atomic<bool> g_rxWorkPending{false};
+    static std::atomic<bool> g_processing{false};
+
+    // The PHY is polled: an external RGMII PHY raises no MAC interrupt on
+    // link changes, and its own interrupt pin is not wired to anything
+    // ACPI describes.
+    static constexpr uint64_t LINK_POLL_MS = 1000;
+    static std::atomic<uint64_t> g_nextLinkPollMs{0};
+    static std::atomic<bool> g_linkUp{false};
+    static uint32_t g_linkSpeed = 0;
+    static bool g_linkFullDuplex = false;
+
+    static uint64_t g_rxPacketCount = 0;
+    static uint64_t g_txPacketCount = 0;
+    static uint64_t g_rxErrorCount = 0;
+
+    static RxCallback g_rxCallback = nullptr;
+
+    // ====================================================================
+    // Register access
+    // ====================================================================
+
+    static uint32_t Read(uint32_t reg) {
+        return *(volatile uint32_t*)(g_regs + reg);
+    }
+
+    static void Write(uint32_t reg, uint32_t value) {
+        *(volatile uint32_t*)(g_regs + reg) = value;
+    }
+
+    static void SetBits(uint32_t reg, uint32_t bits) { Write(reg, Read(reg) | bits); }
+    static void ClearBits(uint32_t reg, uint32_t bits) { Write(reg, Read(reg) & ~bits); }
+
+    static uint32_t DescReg(uint32_t base, uint32_t slot, uint32_t word) {
+        return base + slot * DESC_SIZE + word * 4;
+    }
+
+    // ====================================================================
+    // MDIO (the UniMAC's clause 22 master)
+    // ====================================================================
+
+    // Standard PHY registers and bits.
+    static constexpr uint32_t MII_BMCR      = 0x00;
+    static constexpr uint32_t MII_BMSR      = 0x01;
+    static constexpr uint32_t MII_PHYSID1   = 0x02;
+    static constexpr uint32_t MII_PHYSID2   = 0x03;
+    static constexpr uint32_t MII_ADVERTISE = 0x04;
+    static constexpr uint32_t MII_LPA       = 0x05;
+    static constexpr uint32_t MII_CTRL1000  = 0x09;
+    static constexpr uint32_t MII_STAT1000  = 0x0A;
+
+    static constexpr uint16_t BMCR_SPEED1000 = 1u << 6;
+    static constexpr uint16_t BMCR_FULLDPLX  = 1u << 8;
+    static constexpr uint16_t BMCR_ANRESTART = 1u << 9;
+    static constexpr uint16_t BMCR_ANENABLE  = 1u << 12;
+    static constexpr uint16_t BMCR_SPEED100  = 1u << 13;
+    static constexpr uint16_t BMCR_RESET     = 1u << 15;
+    static constexpr uint16_t BMSR_LSTATUS   = 1u << 2;
+    static constexpr uint16_t BMSR_ANEGCOMPLETE = 1u << 5;
+    static constexpr uint16_t ADVERTISE_CSMA = 0x0001;
+    static constexpr uint16_t ADVERTISE_10HALF  = 1u << 5;
+    static constexpr uint16_t ADVERTISE_10FULL  = 1u << 6;
+    static constexpr uint16_t ADVERTISE_100HALF = 1u << 7;
+    static constexpr uint16_t ADVERTISE_100FULL = 1u << 8;
+    static constexpr uint16_t ADVERTISE_1000HALF = 1u << 8;   // MII_CTRL1000
+    static constexpr uint16_t ADVERTISE_1000FULL = 1u << 9;   // MII_CTRL1000
+    static constexpr uint16_t LPA_1000HALF   = 1u << 10;      // MII_STAT1000
+    static constexpr uint16_t LPA_1000FULL   = 1u << 11;      // MII_STAT1000
+
+    static bool MdioRun(uint32_t command) {
+        Write(MDIO_CMD, command);
+        Write(MDIO_CMD, command | MDIO_START_BUSY);
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Read(MDIO_CMD) & MDIO_START_BUSY) {
+            if (Timekeeping::GetMilliseconds() - start > 20) return false;
+            Arch::Pause();
+        }
+        return true;
+    }
+
+    static bool MdioRead(uint32_t phy, uint32_t reg, uint16_t& value) {
+        if (!MdioRun(MDIO_RD | (phy << MDIO_PMD_SHIFT) | (reg << MDIO_REG_SHIFT))) return false;
+        uint32_t result = Read(MDIO_CMD);
+        if (result & MDIO_READ_FAIL) return false;
+        value = (uint16_t)result;
+        return true;
+    }
+
+    static bool MdioWrite(uint32_t phy, uint32_t reg, uint16_t value) {
+        return MdioRun(MDIO_WR | (phy << MDIO_PMD_SHIFT) | (reg << MDIO_REG_SHIFT) | value);
+    }
+
+    static uint16_t PhyRead(uint32_t reg) {
+        uint16_t value = 0;
+        if (!MdioRead(g_phyAddr, reg, value)) return 0;
+        return value;
+    }
+
+    static bool PhyPresent(uint32_t addr, uint32_t& id) {
+        uint16_t id1 = 0, id2 = 0;
+        if (!MdioRead(addr, MII_PHYSID1, id1) || !MdioRead(addr, MII_PHYSID2, id2)) return false;
+        if ((id1 == 0xFFFF && id2 == 0xFFFF) || (id1 == 0 && id2 == 0)) return false;
+        id = ((uint32_t)id1 << 16) | id2;
+        return true;
+    }
+
+    // The board puts the PHY at address 1; scan the bus if it is not there.
+    static bool FindPhy(uint32_t& id) {
+        if (PhyPresent(1, id)) { g_phyAddr = 1; return true; }
+        for (uint32_t addr = 0; addr < 32; addr++) {
+            if (addr != 1 && PhyPresent(addr, id)) { g_phyAddr = addr; return true; }
+        }
+        return false;
+    }
+
+    // ====================================================================
+    // Link
+    // ====================================================================
+
+    // BMSR latches link failures: the first read reports any loss since the
+    // last one, the second the current state.
+    static bool PhyLinkUp() {
+        (void)PhyRead(MII_BMSR);
+        uint16_t bmsr = PhyRead(MII_BMSR);
+        if (!(bmsr & BMSR_LSTATUS)) return false;
+        uint16_t bmcr = PhyRead(MII_BMCR);
+        return !(bmcr & BMCR_ANENABLE) || (bmsr & BMSR_ANEGCOMPLETE);
+    }
+
+    // The highest mode both ends advertised (or the forced mode).
+    static void ResolveLink(uint32_t& speed, bool& fullDuplex) {
+        uint16_t bmcr = PhyRead(MII_BMCR);
+        if (!(bmcr & BMCR_ANENABLE)) {
+            speed = (bmcr & BMCR_SPEED1000) ? 1000 : (bmcr & BMCR_SPEED100) ? 100 : 10;
+            fullDuplex = (bmcr & BMCR_FULLDPLX) != 0;
+            return;
+        }
+
+        uint16_t ctrl1000 = PhyRead(MII_CTRL1000);
+        uint16_t stat1000 = PhyRead(MII_STAT1000);
+        if ((ctrl1000 & ADVERTISE_1000FULL) && (stat1000 & LPA_1000FULL)) {
+            speed = 1000; fullDuplex = true; return;
+        }
+        if ((ctrl1000 & ADVERTISE_1000HALF) && (stat1000 & LPA_1000HALF)) {
+            speed = 1000; fullDuplex = false; return;
+        }
+
+        uint16_t common = PhyRead(MII_ADVERTISE) & PhyRead(MII_LPA);
+        if (common & ADVERTISE_100FULL)      { speed = 100; fullDuplex = true; }
+        else if (common & ADVERTISE_100HALF) { speed = 100; fullDuplex = false; }
+        else if (common & ADVERTISE_10FULL)  { speed = 10;  fullDuplex = true; }
+        else                                 { speed = 10;  fullDuplex = false; }
+    }
+
+    // Point the RGMII block and the UniMAC at the negotiated mode and start
+    // both directions.
+    static void ConfigureMac(uint32_t speed, bool fullDuplex) {
+        uint32_t oob = Read(EXT_RGMII_OOB_CTRL);
+        oob &= ~OOB_DISABLE;
+        oob |= RGMII_LINK | RGMII_MODE_EN;
+        if (g_phyMode == PhyMode::Rgmii || g_phyMode == PhyMode::RgmiiId) oob |= ID_MODE_DIS;
+        else oob &= ~ID_MODE_DIS;
+        Write(EXT_RGMII_OOB_CTRL, oob);
+
+        uint32_t cmd = (speed == 1000 ? CMD_SPEED_1000 : speed == 100 ? CMD_SPEED_100 : CMD_SPEED_10)
+                       << CMD_SPEED_SHIFT;
+        if (!fullDuplex) cmd |= CMD_HD_EN;
+        Write(UMAC_CMD, cmd);
+        Write(UMAC_CMD, cmd | CMD_TX_EN | CMD_RX_EN);
+    }
+
+    static void PollLink() {
+        bool up = PhyLinkUp();
+        bool wasUp = g_linkUp.load(std::memory_order_relaxed);
+
+        if (up) {
+            uint32_t speed = 0;
+            bool fullDuplex = false;
+            ResolveLink(speed, fullDuplex);
+            if (wasUp && speed == g_linkSpeed && fullDuplex == g_linkFullDuplex) return;
+
+            ConfigureMac(speed, fullDuplex);
+            g_linkSpeed = speed;
+            g_linkFullDuplex = fullDuplex;
+            g_linkUp.store(true, std::memory_order_release);
+            KernelLogStream(OK, "GENET") << "Link up: " << base::dec << (uint64_t)speed
+                << " Mbit/s " << (fullDuplex ? "full" : "half") << " duplex";
+        } else if (wasUp) {
+            g_linkUp.store(false, std::memory_order_release);
+            g_linkSpeed = 0;
+            ClearBits(EXT_RGMII_OOB_CTRL, RGMII_LINK);
+            KernelLogStream(INFO, "GENET") << "Link down";
+        }
+    }
+
+    // Leave a link the firmware already negotiated alone (renegotiating
+    // costs seconds); otherwise reset the PHY and advertise 10/100 and
+    // 1000 full duplex (the MAC has no 1000 half duplex).
+    static void StartPhy() {
+        if (PhyLinkUp()) {
+            KernelLogStream(INFO, "GENET") << "PHY already has a link, keeping it";
+            return;
+        }
+
+        MdioWrite(g_phyAddr, MII_BMCR, BMCR_RESET);
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (PhyRead(MII_BMCR) & BMCR_RESET) {
+            if (Timekeeping::GetMilliseconds() - start > 500) {
+                KernelLogStream(WARNING, "GENET") << "PHY reset timed out";
+                break;
+            }
+            Timekeeping::BusyWaitUs(100);
+        }
+
+        MdioWrite(g_phyAddr, MII_ADVERTISE, ADVERTISE_CSMA | ADVERTISE_10HALF | ADVERTISE_10FULL |
+                                            ADVERTISE_100HALF | ADVERTISE_100FULL);
+        uint16_t ctrl1000 = PhyRead(MII_CTRL1000);
+        ctrl1000 &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL);
+        MdioWrite(g_phyAddr, MII_CTRL1000, ctrl1000 | ADVERTISE_1000FULL);
+        MdioWrite(g_phyAddr, MII_BMCR, BMCR_ANENABLE | BMCR_ANRESTART);
+    }
+
+    // ====================================================================
+    // MAC bring-up
+    // ====================================================================
+
+    static bool WaitDmaDisabled(uint32_t engineRegs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (!(Read(engineRegs + DMA_STATUS) & DMA_DISABLED)) {
+            if (Timekeeping::GetMilliseconds() - start > 5) return false;
+            Arch::Pause();
+        }
+        return true;
+    }
+
+    // Stop both DMA engines (the firmware's network stack may have left
+    // them running) and drop whatever the buffers still hold.
+    static void StopDma() {
+        Write(TDMA_REGS + DMA_CTRL, 0);
+        Write(RDMA_REGS + DMA_CTRL, 0);
+        if (!WaitDmaDisabled(TDMA_REGS)) KernelLogStream(WARNING, "GENET") << "TX DMA did not stop";
+        if (!WaitDmaDisabled(RDMA_REGS)) KernelLogStream(WARNING, "GENET") << "RX DMA did not stop";
+
+        Write(UMAC_TX_FLUSH, 1);
+        Timekeeping::BusyWaitUs(10);
+        Write(UMAC_TX_FLUSH, 0);
+
+        SetBits(SYS_RBUF_FLUSH_CTRL, 1u << 0);
+        Timekeeping::BusyWaitUs(10);
+        ClearBits(SYS_RBUF_FLUSH_CTRL, 1u << 0);
+    }
+
+    // Reset the UniMAC with RGMII loopback on, so its receive clock stays
+    // stable through the reset, and leave it disabled until the link is up.
+    static void ResetUmac() {
+        SetBits(SYS_RBUF_FLUSH_CTRL, 1u << 1);
+        Timekeeping::BusyWaitUs(10);
+        ClearBits(SYS_RBUF_FLUSH_CTRL, 1u << 1);
+        Timekeeping::BusyWaitUs(10);
+        Write(SYS_RBUF_FLUSH_CTRL, 0);
+        Timekeeping::BusyWaitUs(10);
+
+        Write(UMAC_CMD, 0);
+        Write(UMAC_CMD, CMD_SW_RESET | CMD_LCL_LOOP_EN);
+        Timekeeping::BusyWaitUs(2);
+        Write(UMAC_CMD, 0);
+
+        Write(UMAC_MIB_CTRL, MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT);
+        Write(UMAC_MIB_CTRL, 0);
+
+        Write(UMAC_MAX_FRAME_LEN, MAX_FRAME_LEN);
+
+        // No status blocks in front of frames; received frames start two
+        // bytes into their buffer, so the IP header lands 4-byte aligned.
+        uint32_t rbuf = Read(RBUF_CTRL);
+        rbuf &= ~RBUF_64B_EN;
+        rbuf |= RBUF_ALIGN_2B;
+        Write(RBUF_CTRL, rbuf);
+        ClearBits(TBUF_CTRL, TBUF_64B_EN);
+        Write(RBUF_TBUF_SIZE_CTRL, 1);
+    }
+
+    static void ReadFirmwareMac(uint8_t mac[6]) {
+        uint32_t mac0 = Read(UMAC_MAC0);
+        uint32_t mac1 = Read(UMAC_MAC1);
+        mac[0] = (uint8_t)(mac0 >> 24);
+        mac[1] = (uint8_t)(mac0 >> 16);
+        mac[2] = (uint8_t)(mac0 >> 8);
+        mac[3] = (uint8_t)mac0;
+        mac[4] = (uint8_t)(mac1 >> 8);
+        mac[5] = (uint8_t)mac1;
+    }
+
+    static bool MacUsable(const uint8_t mac[6]) {
+        bool allZero = true, allOnes = true;
+        for (int i = 0; i < 6; i++) {
+            if (mac[i] != 0x00) allZero = false;
+            if (mac[i] != 0xFF) allOnes = false;
+        }
+        return !allZero && !allOnes && !(mac[0] & 1);
+    }
+
+    static void SetMdfEntry(uint32_t& index, const uint8_t mac[6]) {
+        Write(UMAC_MDF_ADDR + index * 4, ((uint32_t)mac[0] << 8) | mac[1]);
+        Write(UMAC_MDF_ADDR + (index + 1) * 4, ((uint32_t)mac[2] << 24) | ((uint32_t)mac[3] << 16) |
+                                               ((uint32_t)mac[4] << 8) | mac[5]);
+        index += 2;
+    }
+
+    // Station address, and the destination filter: broadcast and our own
+    // address. Filter n is enabled by bit (16 - n).
+    static void ProgramAddress() {
+        Write(UMAC_MAC0, ((uint32_t)g_mac[0] << 24) | ((uint32_t)g_mac[1] << 16) |
+                         ((uint32_t)g_mac[2] << 8) | g_mac[3]);
+        Write(UMAC_MAC1, ((uint32_t)g_mac[4] << 8) | g_mac[5]);
+
+        static constexpr uint8_t Broadcast[6] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
+        uint32_t index = 0;
+        SetMdfEntry(index, Broadcast);
+        SetMdfEntry(index, g_mac);
+        uint32_t filters = index / 2;
+        uint32_t enable = 0;
+        for (uint32_t i = 0; i < filters; i++) enable |= 1u << (MAX_MDF_FILTER - 1 - i);
+        Write(UMAC_MDF_CTRL, enable);
+    }
+
+    static bool AllocateBuffers(uint8_t** bufs, uint32_t count) {
+        constexpr uint32_t PerPage = 0x1000 / BUF_LENGTH;
+        for (uint32_t i = 0; i < count; i += PerPage) {
+            uint8_t* page = (uint8_t*)Memory::g_pfa->AllocateZeroed();
+            if (page == nullptr) return false;
+            // The zeroes sit dirty in the cache; write them back now so no
+            // eviction lands on top of a frame the MAC wrote later.
+            Memory::Dma::SyncForDevice(page, 0x1000, g_coherent);
+            for (uint32_t j = 0; j < PerPage && i + j < count; j++) bufs[i + j] = page + j * BUF_LENGTH;
+        }
+        return true;
+    }
+
+    // Ring 16 spans all 256 descriptors (START_ADDR/END_ADDR count 32-bit
+    // words). The producer/consumer counters are zeroed, but if the MAC
+    // keeps an old value the software side follows it instead: the slot is
+    // always counter % 256.
+    static void InitRxRing(uint32_t burst) {
+        Write(RDMA_REGS + DMA_SCB_BURST_SIZE, burst);
+
+        Write(RDMA_RING + RDMA_PROD_INDEX, 0);
+        Write(RDMA_RING + RDMA_CONS_INDEX, 0);
+        Write(RDMA_RING + DMA_START_ADDR, 0);
+        Write(RDMA_RING + RDMA_READ_PTR, 0);
+        Write(RDMA_RING + RDMA_WRITE_PTR, 0);
+        Write(RDMA_RING + DMA_END_ADDR, RX_DESCS * DESC_SIZE / 4 - 1);
+        Write(RDMA_RING + DMA_RING_BUF_SIZE, (RX_DESCS << 16) | BUF_LENGTH);
+        Write(RDMA_RING + RDMA_XON_XOFF_THRESH, (5u << 16) | (RX_DESCS >> 4));
+        Write(RDMA_RING + DMA_MBUF_DONE_THRESH, 1);
+
+        uint32_t cons = Read(RDMA_RING + RDMA_PROD_INDEX) & 0xFFFF;
+        Write(RDMA_RING + RDMA_CONS_INDEX, cons);
+        g_rxCons.store(cons, std::memory_order_relaxed);
+        if (cons != 0) {
+            KernelLogStream(INFO, "GENET") << "RX ring resumes at index " << base::dec << (uint64_t)cons;
+        }
+
+        for (uint32_t i = 0; i < RX_DESCS; i++) {
+            uint64_t phys = Memory::SubHHDM(g_rxBufs[i]);
+            Write(DescReg(RX_DESC_BASE, i, 1), (uint32_t)phys);
+            Write(DescReg(RX_DESC_BASE, i, 2), (uint32_t)(phys >> 32));
+            Write(DescReg(RX_DESC_BASE, i, 0), (BUF_LENGTH << DMA_BUFLENGTH_SHIFT) | DMA_OWN);
+        }
+
+        Write(RDMA_REGS + DMA_RING_CFG, 1u << DEFAULT_Q);
+    }
+
+    static void InitTxRing(uint32_t burst) {
+        Write(TDMA_REGS + DMA_SCB_BURST_SIZE, burst);
+
+        Write(TDMA_RING + TDMA_PROD_INDEX, 0);
+        Write(TDMA_RING + TDMA_CONS_INDEX, 0);
+        Write(TDMA_RING + DMA_START_ADDR, 0);
+        Write(TDMA_RING + TDMA_READ_PTR, 0);
+        Write(TDMA_RING + TDMA_WRITE_PTR, 0);
+        Write(TDMA_RING + DMA_END_ADDR, TX_DESCS * DESC_SIZE / 4 - 1);
+        Write(TDMA_RING + DMA_RING_BUF_SIZE, (TX_DESCS << 16) | BUF_LENGTH);
+        Write(TDMA_RING + DMA_MBUF_DONE_THRESH, 1);
+        Write(TDMA_RING + TDMA_FLOW_PERIOD, 0);
+
+        g_txProd = Read(TDMA_RING + TDMA_CONS_INDEX) & 0xFFFF;
+        Write(TDMA_RING + TDMA_PROD_INDEX, g_txProd);
+        if (g_txProd != 0) {
+            KernelLogStream(INFO, "GENET") << "TX ring resumes at index " << base::dec << (uint64_t)g_txProd;
+        }
+
+        Write(TDMA_REGS + DMA_RING_CFG, 1u << DEFAULT_Q);
+    }
+
+    static void StartDma() {
+        uint32_t ctrl = (1u << (DEFAULT_Q + DMA_RING_BUF_EN_SHIFT)) | DMA_EN;
+        Write(TDMA_REGS + DMA_CTRL, ctrl);
+        Write(RDMA_REGS + DMA_CTRL, ctrl);
+    }
+
+    // ====================================================================
+    // Interrupt
+    // ====================================================================
+
+    // Acknowledge and mask; the frames are dispatched from the bottom half,
+    // which takes process-context locks.
+    static void HandleInterrupt(uint8_t, bool) {
+        if (!g_initialized) return;
+        uint32_t status = Read(INTRL2_0 + INTRL2_CPU_STAT) & ~Read(INTRL2_0 + INTRL2_CPU_MASK_STATUS);
+        if (status == 0) return;
+        Write(INTRL2_0 + INTRL2_CPU_CLEAR, status);
+
+        if (status & UMAC_IRQ_RXDMA_DONE) {
+            Write(INTRL2_0 + INTRL2_CPU_MASK_SET, UMAC_IRQ_RXDMA_DONE);
+            g_rxWorkPending.store(true, std::memory_order_release);
+        }
+    }
+
+    // ====================================================================
+    // Receive
+    // ====================================================================
+
+    static void PollRx() {
+        uint32_t cons = g_rxCons.load(std::memory_order_relaxed);
+        uint32_t prod = Read(RDMA_RING + RDMA_PROD_INDEX) & 0xFFFF;
+
+        // At most one revolution per pass; the bottom half comes back for
+        // the rest.
+        uint32_t processed = 0;
+        while (cons != prod && processed < RX_DESCS) {
+            Arch::DmaReadFence();
+            uint32_t slot = cons % RX_DESCS;
+            uint32_t lenStat = Read(DescReg(RX_DESC_BASE, slot, 0));
+            uint32_t length = (lenStat >> DMA_BUFLENGTH_SHIFT) & DMA_BUFLENGTH_MASK;
+            uint8_t* buffer = g_rxBufs[slot];
+
+            bool whole = (lenStat & (DMA_SOP | DMA_EOP)) == (DMA_SOP | DMA_EOP);
+            if (!whole || (lenStat & DMA_RX_ERRORS) || length <= RX_BUF_OFFSET || length > BUF_LENGTH) {
+                g_rxErrorCount++;
+            } else {
+                Memory::Dma::SyncForCpu(buffer, length, g_coherent);
+                g_rxPacketCount++;
+                if (g_rxCallback != nullptr) {
+                    g_rxCallback(buffer + RX_BUF_OFFSET, (uint16_t)(length - RX_BUF_OFFSET));
+                }
+            }
+
+            cons = (cons + 1) & 0xFFFF;
+            g_rxCons.store(cons, std::memory_order_relaxed);
+            Write(RDMA_RING + RDMA_CONS_INDEX, cons);
+            processed++;
+            if (cons == prod) prod = Read(RDMA_RING + RDMA_PROD_INDEX) & 0xFFFF;
+        }
+    }
+
+    static bool RxPending() {
+        uint32_t prod = Read(RDMA_RING + RDMA_PROD_INDEX) & 0xFFFF;
+        return prod != g_rxCons.load(std::memory_order_relaxed);
+    }
+
+    // ====================================================================
+    // Public API
+    // ====================================================================
+
+    bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
+                       bool activeLow, bool dmaCoherent, PhyMode phyMode, uint32_t burst) {
+        if (g_initialized) return false;
+
+        if (mmioSize < 0x10000) mmioSize = 0x10000;
+        for (uint64_t offset = 0; offset < mmioSize; offset += 0x1000) {
+            Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
+        }
+        g_regs = (volatile uint8_t*)Memory::HHDM(mmioPhys);
+        g_coherent = dmaCoherent;
+        g_phyMode = phyMode;
+        if (burst == 0) burst = 8;
+
+        // GENET v5 reports major revision 6 (v4 reports 5).
+        uint32_t rev = Read(SYS_REV_CTRL);
+        uint32_t major = (rev >> 24) & 0x0F;
+        if (major != 6) {
+            KernelLogStream(WARNING, "GENET") << "Unsupported revision 0x" << base::hex << (uint64_t)rev;
+            return false;
+        }
+
+        // Mask everything while the controller is reconfigured.
+        Write(INTRL2_0 + INTRL2_CPU_MASK_SET, 0xFFFFFFFF);
+        Write(INTRL2_0 + INTRL2_CPU_CLEAR, 0xFFFFFFFF);
+        Write(INTRL2_1 + INTRL2_CPU_MASK_SET, 0xFFFFFFFF);
+        Write(INTRL2_1 + INTRL2_CPU_CLEAR, 0xFFFFFFFF);
+
+        // The board's address comes from OTP through the firmware; else
+        // whatever the firmware's own driver programmed.
+        uint8_t firmwareMac[6];
+        ReadFirmwareMac(firmwareMac);
+        const char* macSource = "OTP";
+        if (!Platform::RpiFirmware::GetBoardMacAddress(g_mac) || !MacUsable(g_mac)) {
+            if (MacUsable(firmwareMac)) {
+                for (int i = 0; i < 6; i++) g_mac[i] = firmwareMac[i];
+                macSource = "firmware";
+            } else {
+                // Locally administered, fixed so DHCP leases survive reboots.
+                static constexpr uint8_t Fallback[6] = { 0x02, 0x4D, 0x54, 0x4B, 0x00, 0x01 };
+                for (int i = 0; i < 6; i++) g_mac[i] = Fallback[i];
+                macSource = "fallback";
+            }
+        }
+
+        Write(SYS_PORT_CTRL, PORT_MODE_EXT_GPHY);
+        StopDma();
+        ResetUmac();
+        ProgramAddress();
+
+        if (!AllocateBuffers(g_rxBufs, RX_DESCS) || !AllocateBuffers(g_txBufs, TX_DESCS)) {
+            KernelLogStream(ERROR, "GENET") << "Out of memory for packet buffers";
+            return false;
+        }
+        InitRxRing(burst);
+        InitTxRing(burst);
+        StartDma();
+
+        uint32_t phyId = 0;
+        if (!FindPhy(phyId)) {
+            KernelLogStream(ERROR, "GENET") << "No PHY answers on MDIO";
+            StopDma();
+            return false;
+        }
+
+        KernelLogStream(OK, "GENET") << "GENET v5." << base::dec << (uint64_t)((rev >> 16) & 0x0F)
+            << " at 0x" << base::hex << mmioPhys << ", PHY 0x" << (uint64_t)phyId
+            << " at MDIO " << base::dec << (uint64_t)g_phyAddr
+            << (dmaCoherent ? ", coherent DMA" : ", non-coherent DMA");
+        KernelLogStream(OK, "GENET") << "MAC " << base::hex
+            << (uint64_t)g_mac[0] << ":" << (uint64_t)g_mac[1] << ":" << (uint64_t)g_mac[2] << ":"
+            << (uint64_t)g_mac[3] << ":" << (uint64_t)g_mac[4] << ":" << (uint64_t)g_mac[5]
+            << " (" << macSource << ")";
+
+        // RGMII on with the board's delays; the link (RGMII_LINK, speed,
+        // TX/RX enable) follows from the PHY poll.
+        uint32_t oob = Read(EXT_RGMII_OOB_CTRL);
+        oob &= ~(OOB_DISABLE | RGMII_LINK);
+        oob |= RGMII_MODE_EN;
+        if (phyMode == PhyMode::Rgmii || phyMode == PhyMode::RgmiiId) oob |= ID_MODE_DIS;
+        else oob &= ~ID_MODE_DIS;
+        Write(EXT_RGMII_OOB_CTRL, oob);
+
+        StartPhy();
+
+        uint8_t line = gsi != 0 ? Hal::GsiRouting::LineForGsi(gsi, levelTriggered, activeLow) : 0;
+        if (line != 0) {
+            Arch::RegisterIrqHandler(line, HandleInterrupt);
+            Arch::EnableLegacyIrq(line);
+            KernelLogStream(OK, "GENET") << "Interrupt: GSI " << base::dec << (uint64_t)gsi
+                << " on platform line " << (uint64_t)line;
+        } else {
+            KernelLogStream(WARNING, "GENET") << "No interrupt, receive is polled";
+        }
+
+        g_initialized = true;
+        Write(INTRL2_0 + INTRL2_CPU_MASK_CLEAR, UMAC_IRQ_RXDMA_DONE);
+
+        PollLink();
+        g_nextLinkPollMs.store(Timekeeping::GetMilliseconds() + LINK_POLL_MS, std::memory_order_relaxed);
+        return true;
+    }
+
+    // Frames are copied into the slot's buffer, so the caller's buffer is
+    // free again on return. Fails while the ring is full or the link down.
+    bool SendPacket(const uint8_t* data, uint16_t length) {
+        if (!g_initialized || data == nullptr || length == 0 || length > MAX_FRAME_LEN) return false;
+        if (!g_linkUp.load(std::memory_order_acquire)) return false;
+
+        g_txLock.Acquire();
+
+        uint32_t cons = Read(TDMA_RING + TDMA_CONS_INDEX) & 0xFFFF;
+        if (((g_txProd - cons) & 0xFFFF) >= TX_DESCS - 1) {
+            g_txLock.Release();
+            return false;
+        }
+
+        uint32_t slot = g_txProd % TX_DESCS;
+        uint8_t* buffer = g_txBufs[slot];
+        memcpy(buffer, data, length);
+        // The MAC sends short frames as given; pad to the 60-byte minimum.
+        uint32_t frameLength = length;
+        if (frameLength < 60) {
+            memset(buffer + frameLength, 0, 60 - frameLength);
+            frameLength = 60;
+        }
+        Memory::Dma::SyncForDevice(buffer, frameLength, g_coherent);
+
+        uint64_t phys = Memory::SubHHDM(buffer);
+        Write(DescReg(TX_DESC_BASE, slot, 1), (uint32_t)phys);
+        Write(DescReg(TX_DESC_BASE, slot, 2), (uint32_t)(phys >> 32));
+        Write(DescReg(TX_DESC_BASE, slot, 0), (frameLength << DMA_BUFLENGTH_SHIFT) |
+              (DMA_TX_QTAG_MASK << DMA_TX_QTAG_SHIFT) | DMA_TX_APPEND_CRC | DMA_SOP | DMA_EOP);
+
+        g_txProd = (g_txProd + 1) & 0xFFFF;
+        Arch::DmaWriteFence();
+        Write(TDMA_RING + TDMA_PROD_INDEX, g_txProd);
+        g_txPacketCount++;
+
+        g_txLock.Release();
+        return true;
+    }
+
+    const uint8_t* GetMacAddress() {
+        return g_mac;
+    }
+
+    bool IsInitialized() {
+        return g_initialized;
+    }
+
+    bool IsLinkUp() {
+        return g_initialized && g_linkUp.load(std::memory_order_acquire);
+    }
+
+    uint32_t GetLinkSpeed() {
+        return IsLinkUp() ? g_linkSpeed : 0;
+    }
+
+    uint64_t GetRxPacketCount() {
+        return g_rxPacketCount;
+    }
+
+    uint64_t GetTxPacketCount() {
+        return g_txPacketCount;
+    }
+
+    void SetRxCallback(RxCallback callback) {
+        g_rxCallback = callback;
+    }
+
+    bool HasDeferredWork() {
+        if (!g_initialized) return false;
+        return g_rxWorkPending.load(std::memory_order_acquire) || RxPending() ||
+               Timekeeping::GetMilliseconds() >= g_nextLinkPollMs.load(std::memory_order_relaxed);
+    }
+
+    void ProcessDeferredWork() {
+        if (!g_initialized) return;
+
+        bool expected = false;
+        if (!g_processing.compare_exchange_strong(expected, true,
+                std::memory_order_acquire, std::memory_order_relaxed)) return;
+
+        uint64_t now = Timekeeping::GetMilliseconds();
+        if (now >= g_nextLinkPollMs.load(std::memory_order_relaxed)) {
+            g_nextLinkPollMs.store(now + LINK_POLL_MS, std::memory_order_relaxed);
+            PollLink();
+        }
+
+        g_rxWorkPending.store(false, std::memory_order_release);
+        PollRx();
+
+        // A frame that arrived while masked has latched the status bit, so
+        // unmasking raises the interrupt again rather than losing it.
+        Write(INTRL2_0 + INTRL2_CPU_MASK_CLEAR, UMAC_IRQ_RXDMA_DONE);
+
+        g_processing.store(false, std::memory_order_release);
+    }
+
+}

kernel/src/Drivers/Net/Genet.hpp

@@ -0,0 +1,197 @@
+/*
+    * Genet.hpp
+    * Broadcom GENET v5 Ethernet driver (Raspberry Pi 4 / 400)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// The BCM2711's on-chip Gigabit Ethernet MAC (GENET v5, a UniMAC behind a
+// DMA engine), wired over RGMII to a BCM54213PE PHY at MDIO address 1. The
+// Pi Firmware Task Force UEFI firmware describes it as BCM6E4E
+// (\_SB.ETH0): one register block, two interrupts (the first carries the
+// default queue), _CCA = 0 (buffers do not snoop the CPU caches) and a
+// _DSD with "phy-mode" and "brcm,max-dma-burst-size".
+//
+// Only the default queue (ring 16) is used, with all 256 descriptors of
+// each direction. The descriptors live in the MAC's own SRAM, so only the
+// packet buffers need cache maintenance.
+
+namespace Drivers::Net::Genet {
+
+    // ---- Register map (offsets from the block base) ----
+
+    constexpr uint32_t SYS_REV_CTRL        = 0x0000;
+    constexpr uint32_t SYS_PORT_CTRL       = 0x0004;
+    constexpr uint32_t SYS_RBUF_FLUSH_CTRL = 0x0008;
+    constexpr uint32_t SYS_TBUF_FLUSH_CTRL = 0x000C;
+
+    constexpr uint32_t EXT_RGMII_OOB_CTRL  = 0x008C;
+
+    constexpr uint32_t INTRL2_0            = 0x0200;   // default queue, MAC events
+    constexpr uint32_t INTRL2_1            = 0x0240;   // priority queues (unused)
+    constexpr uint32_t INTRL2_CPU_STAT       = 0x00;
+    constexpr uint32_t INTRL2_CPU_CLEAR      = 0x08;
+    constexpr uint32_t INTRL2_CPU_MASK_STATUS = 0x0C;
+    constexpr uint32_t INTRL2_CPU_MASK_SET   = 0x10;
+    constexpr uint32_t INTRL2_CPU_MASK_CLEAR = 0x14;
+
+    constexpr uint32_t RBUF_CTRL           = 0x0300;
+    constexpr uint32_t RBUF_TBUF_SIZE_CTRL = 0x03B4;
+    constexpr uint32_t TBUF_CTRL           = 0x0600;
+
+    constexpr uint32_t UMAC_CMD            = 0x0808;
+    constexpr uint32_t UMAC_MAC0           = 0x080C;
+    constexpr uint32_t UMAC_MAC1           = 0x0810;
+    constexpr uint32_t UMAC_MAX_FRAME_LEN  = 0x0814;
+    constexpr uint32_t UMAC_TX_FLUSH       = 0x0B34;
+    constexpr uint32_t UMAC_MIB_CTRL       = 0x0D80;
+    constexpr uint32_t MDIO_CMD            = 0x0E14;
+    constexpr uint32_t UMAC_MDF_CTRL       = 0x0E50;
+    constexpr uint32_t UMAC_MDF_ADDR       = 0x0E54;
+
+    // Descriptor SRAM, then the DMA registers: 17 rings of 0x40, then the
+    // engine's own registers.
+    constexpr uint32_t TOTAL_DESCS         = 256;
+    constexpr uint32_t DESC_SIZE           = 12;       // length/status, address lo, address hi
+    constexpr uint32_t RX_DESC_BASE        = 0x2000;
+    constexpr uint32_t TX_DESC_BASE        = 0x4000;
+    constexpr uint32_t RDMA_BASE           = RX_DESC_BASE + TOTAL_DESCS * DESC_SIZE;
+    constexpr uint32_t TDMA_BASE           = TX_DESC_BASE + TOTAL_DESCS * DESC_SIZE;
+    constexpr uint32_t DEFAULT_Q           = 16;
+    constexpr uint32_t DMA_RING_SIZE       = 0x40;
+    constexpr uint32_t RDMA_RING           = RDMA_BASE + DEFAULT_Q * DMA_RING_SIZE;
+    constexpr uint32_t TDMA_RING           = TDMA_BASE + DEFAULT_Q * DMA_RING_SIZE;
+    constexpr uint32_t RDMA_REGS           = RDMA_BASE + (DEFAULT_Q + 1) * DMA_RING_SIZE;
+    constexpr uint32_t TDMA_REGS           = TDMA_BASE + (DEFAULT_Q + 1) * DMA_RING_SIZE;
+
+    // Per-ring registers. RX: the MAC produces, software consumes; TX the
+    // other way round. Both indices are 16-bit counters.
+    constexpr uint32_t RDMA_WRITE_PTR       = 0x00;
+    constexpr uint32_t RDMA_PROD_INDEX      = 0x08;
+    constexpr uint32_t RDMA_CONS_INDEX      = 0x0C;
+    constexpr uint32_t RDMA_XON_XOFF_THRESH = 0x28;
+    constexpr uint32_t RDMA_READ_PTR        = 0x2C;
+    constexpr uint32_t TDMA_READ_PTR        = 0x00;
+    constexpr uint32_t TDMA_CONS_INDEX      = 0x08;
+    constexpr uint32_t TDMA_PROD_INDEX      = 0x0C;
+    constexpr uint32_t TDMA_FLOW_PERIOD     = 0x28;
+    constexpr uint32_t TDMA_WRITE_PTR       = 0x2C;
+    constexpr uint32_t DMA_RING_BUF_SIZE    = 0x10;
+    constexpr uint32_t DMA_START_ADDR       = 0x14;
+    constexpr uint32_t DMA_END_ADDR         = 0x1C;
+    constexpr uint32_t DMA_MBUF_DONE_THRESH = 0x24;
+
+    // Engine registers.
+    constexpr uint32_t DMA_RING_CFG         = 0x00;
+    constexpr uint32_t DMA_CTRL             = 0x04;
+    constexpr uint32_t DMA_STATUS           = 0x08;
+    constexpr uint32_t DMA_SCB_BURST_SIZE   = 0x0C;
+
+    // ---- Register bits ----
+
+    constexpr uint32_t PORT_MODE_EXT_GPHY  = 3;
+
+    constexpr uint32_t RGMII_LINK          = 1u << 4;
+    constexpr uint32_t OOB_DISABLE         = 1u << 5;
+    constexpr uint32_t RGMII_MODE_EN       = 1u << 6;
+    constexpr uint32_t ID_MODE_DIS         = 1u << 16;   // no MAC-side TX clock delay
+
+    constexpr uint32_t RBUF_64B_EN         = 1u << 0;    // receive status block
+    constexpr uint32_t RBUF_ALIGN_2B       = 1u << 1;    // 2 pad bytes before each frame
+    constexpr uint32_t TBUF_64B_EN         = 1u << 0;    // transmit status block
+
+    constexpr uint32_t CMD_TX_EN           = 1u << 0;
+    constexpr uint32_t CMD_RX_EN           = 1u << 1;
+    constexpr uint32_t CMD_SPEED_SHIFT     = 2;
+    constexpr uint32_t CMD_SPEED_10        = 0;
+    constexpr uint32_t CMD_SPEED_100       = 1;
+    constexpr uint32_t CMD_SPEED_1000      = 2;
+    constexpr uint32_t CMD_PROMISC         = 1u << 4;
+    constexpr uint32_t CMD_HD_EN           = 1u << 10;
+    constexpr uint32_t CMD_SW_RESET        = 1u << 13;
+    constexpr uint32_t CMD_LCL_LOOP_EN     = 1u << 15;
+
+    constexpr uint32_t MIB_RESET_RX        = 1u << 0;
+    constexpr uint32_t MIB_RESET_RUNT      = 1u << 1;
+    constexpr uint32_t MIB_RESET_TX        = 1u << 2;
+
+    constexpr uint32_t MDIO_START_BUSY     = 1u << 29;
+    constexpr uint32_t MDIO_READ_FAIL      = 1u << 28;
+    constexpr uint32_t MDIO_RD             = 2u << 26;
+    constexpr uint32_t MDIO_WR             = 1u << 26;
+    constexpr uint32_t MDIO_PMD_SHIFT      = 21;
+    constexpr uint32_t MDIO_REG_SHIFT      = 16;
+
+    constexpr uint32_t MAX_MDF_FILTER      = 17;
+
+    constexpr uint32_t DMA_EN              = 1u << 0;
+    constexpr uint32_t DMA_RING_BUF_EN_SHIFT = 1;
+    constexpr uint32_t DMA_DISABLED        = 1u << 0;    // DMA_STATUS
+
+    constexpr uint32_t UMAC_IRQ_RXDMA_DONE = 1u << 13;   // ring 16 reached its done threshold
+
+    // Descriptor length/status word: length in 27:16, flags in 15:0.
+    constexpr uint32_t DMA_BUFLENGTH_SHIFT = 16;
+    constexpr uint32_t DMA_BUFLENGTH_MASK  = 0x0FFF;
+    constexpr uint32_t DMA_OWN             = 0x8000;
+    constexpr uint32_t DMA_EOP             = 0x4000;
+    constexpr uint32_t DMA_SOP             = 0x2000;
+    constexpr uint32_t DMA_TX_APPEND_CRC   = 0x0040;
+    constexpr uint32_t DMA_TX_QTAG_SHIFT   = 7;
+    constexpr uint32_t DMA_TX_QTAG_MASK    = 0x3F;
+    constexpr uint32_t DMA_RX_LG           = 0x0010;
+    constexpr uint32_t DMA_RX_NO           = 0x0008;
+    constexpr uint32_t DMA_RX_RXER         = 0x0004;
+    constexpr uint32_t DMA_RX_CRC_ERROR    = 0x0002;
+    constexpr uint32_t DMA_RX_OV           = 0x0001;
+    constexpr uint32_t DMA_RX_ERRORS       = DMA_RX_LG | DMA_RX_NO | DMA_RX_RXER |
+                                             DMA_RX_CRC_ERROR | DMA_RX_OV;
+
+    // ---- Driver parameters ----
+
+    constexpr uint32_t RX_DESCS            = TOTAL_DESCS;
+    constexpr uint32_t TX_DESCS            = TOTAL_DESCS;
+    constexpr uint32_t BUF_LENGTH          = 2048;
+    constexpr uint32_t RX_BUF_OFFSET       = 2;          // RBUF_ALIGN_2B padding
+    // Payload 1500 + header 14 + VLAN 4 + Broadcom tag 6 + FCS 4 + pad 8.
+    constexpr uint32_t MAX_FRAME_LEN       = 1536;
+
+    // How the board wires RGMII clock delays (ACPI _DSD "phy-mode").
+    enum class PhyMode : uint8_t {
+        Rgmii,        // no delays from the PHY or the MAC
+        RgmiiId,      // the PHY delays both clocks
+        RgmiiRxid,    // the PHY delays RX, the MAC delays TX (Raspberry Pi 4)
+        RgmiiTxid,
+    };
+
+    // Take over the controller at `mmioPhys`. `gsi` is the default-queue
+    // interrupt (0: poll only). `burst` is the DMA burst size in 32-bit
+    // words from "brcm,max-dma-burst-size" (0: 8).
+    bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
+                       bool activeLow, bool dmaCoherent, PhyMode phyMode, uint32_t burst);
+
+    bool SendPacket(const uint8_t* data, uint16_t length);
+    const uint8_t* GetMacAddress();
+    bool IsInitialized();
+
+    // Carrier as of the last PHY poll (once a second from deferred work).
+    bool IsLinkUp();
+
+    // Negotiated speed in Mbit/s, 0 while the link is down.
+    uint32_t GetLinkSpeed();
+
+    uint64_t GetRxPacketCount();
+    uint64_t GetTxPacketCount();
+
+    using RxCallback = void(*)(const uint8_t* data, uint16_t length);
+    void SetRxCallback(RxCallback callback);
+
+    // The interrupt only masks and flags; frames are dispatched and the PHY
+    // polled from the idle bottom half. HasDeferredWork also looks at the
+    // ring itself, so receive keeps working if the interrupt never fires.
+    bool HasDeferredWork();
+    void ProcessDeferredWork();
+
+}

kernel/src/Drivers/Net/Wifi/Brcmf.hpp

@@ -0,0 +1,218 @@
+/*
+    * Brcmf.hpp
+    * Broadcom/Cypress FullMAC Wi-Fi over SDIO (CYW43455 / CYW43456 on the
+    * Raspberry Pi 4 / 400): driver-internal interface
+    * Derived in part from Linux's brcmfmac, Copyright (c) 2010-2014 Broadcom
+    * Corporation, ISC license (notice in programs/data/licenses/NOTICES.txt).
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+#include <CppLib/Spinlock.hpp>
+
+// A FullMAC radio runs the 802.11 MLME itself: the host asks for scans and
+// joins through ioctls and sees Ethernet frames. Layout, after Linux's
+// brcmfmac:
+//
+//   BrcmfSdio.cpp     the SD controller as an SDIO host (CMD52/CMD53)
+//   BrcmfChip.cpp     backplane access, core enumeration (EROM), clocks,
+//                     firmware + NVRAM download, F2 bring-up, firmware console
+//   BrcmfProto.cpp    SDPCM framing, CDC ioctls, BDC data, the RX pump,
+//                     firmware events
+//   BrcmfConnect.cpp  preinit, CLM, escan, join, keys, the facade entry points
+//
+// The handshake runs on the host (Wpa.cpp), exactly as for the Intel
+// adapter: EAPOL frames arrive as Ethernet frames and the keys go back to the
+// firmware with "wsec_key".
+//
+// Threading: everything on the bus holds g_brcmf.BusLock for the length of one
+// SDIO exchange. The RX pump (BrcmfPoll) has a reentrancy guard and releases
+// the bus lock before dispatching, so the network stack may transmit from
+// inside a delivered frame. Dispatch must never issue an ioctl -- its answer
+// is read by the very pump it would be waiting inside. Work needing one is
+// queued for BrcmfConnectService().
+
+namespace Drivers::Net::Wifi {
+
+    // Facade-visible adapter state; values are the WIFI_STATE_* ABI ones.
+    enum class BrcmfState : uint8_t {
+        Absent = 0,
+        Detected,
+        Booting,
+        Running,
+        Error,
+    };
+
+    struct BrcmfCore {
+        uint16_t Id = 0;
+        uint8_t  Rev = 0;
+        uint32_t Base = 0;
+        uint32_t Wrap = 0;
+    };
+
+    struct BrcmfDevice {
+        BrcmfState State = BrcmfState::Absent;
+        uint64_t   ArasanMmio = 0;
+
+        // Chip
+        uint16_t ChipId = 0;
+        uint8_t  ChipRev = 0;
+        const char* FwName = nullptr;      // "43455" / "43456"
+        BrcmfCore Cores[16];
+        uint32_t CoreCount = 0;
+        uint32_t RamBase = 0;
+        uint32_t RamSize = 0;
+        uint32_t SdioCoreBase = 0;
+        uint32_t PmuBase = 0;
+        bool     SrEnabled = false;
+        uint32_t SharedAddr = 0;           // sdpcm_shared, 0 = not read yet
+        uint32_t ConsoleIdx = 0;           // console bytes already printed
+
+        // Bus
+        kcp::Mutex BusLock;
+        uint32_t Window = 0xFFFFFFFF;      // backplane window, 0xFFFFFFFF = unknown
+        uint32_t HostIntMask = 0;
+
+        // SDPCM
+        uint8_t TxSeq = 0;
+        uint8_t TxMax = 1;
+        bool    RxPending = false;
+
+        // Firmware
+        uint8_t  Mac[6] = {};
+        char     FwVersion[32] = {};
+        uint16_t ChannelCount = 0;
+        bool     Band24 = true;
+        bool     Band5 = false;
+
+        // Health
+        uint32_t FwErrors = 0;
+        uint32_t CmdFailBucket = 0;        // leaky bucket of unanswered ioctls
+        uint64_t RxPackets = 0;
+        uint64_t TxPackets = 0;
+        uint64_t LastPollMs = 0;
+    };
+
+    extern BrcmfDevice g_brcmf;
+
+    // =========================================================================
+    // BrcmfChip.cpp
+    // =========================================================================
+
+    bool BrcmfBpRead32(uint32_t addr, uint32_t& value);
+    bool BrcmfBpWrite32(uint32_t addr, uint32_t value);
+    bool BrcmfBpReadMem(uint32_t addr, uint8_t* buffer, uint32_t len);
+    bool BrcmfBpWriteMem(uint32_t addr, const uint8_t* buffer, uint32_t len);
+    // Point the backplane window at `addr` (function 2 transfers want it on
+    // ChipCommon, as Linux keeps it).
+    bool BrcmfBpSelectWindow(uint32_t addr);
+
+    // Function-1 SDIO-core registers in the 0x1000x space.
+    bool BrcmfF1Read8(uint32_t reg, uint8_t& value);
+    bool BrcmfF1Write8(uint32_t reg, uint8_t value);
+
+    // Power the radio, enumerate the SDIO card, identify the chip, download
+    // firmware + NVRAM, start the ARM and bring up function 2. Leaves
+    // g_brcmf ready for SDPCM traffic.
+    bool BrcmfChipBringUp();
+
+    // Hold the radio in reset (WL_ON low).
+    void BrcmfChipPowerOff();
+
+    // Print what the firmware wrote to its console since the last call, and
+    // a trap/assert summary if it has one. Takes the bus lock.
+    void BrcmfDumpConsole();
+
+    // Load 0:/os/firmware/brcm/brcmfmac<chip>-sdio.<ext>. The caller frees.
+    uint8_t* BrcmfLoadFile(const char* ext, uint32_t& size, bool required);
+
+    // =========================================================================
+    // BrcmfProto.cpp
+    // =========================================================================
+
+    void BrcmfProtoReset();
+
+    // Drain the device: interrupt status, mailbox, every pending frame, and
+    // queued transmits. Reentrancy guarded; returns at once if another core
+    // is already pumping.
+    void BrcmfPoll();
+
+    // Whether a poll is due (card interrupt, queued TX, the periodic sweep).
+    bool BrcmfPollDue(uint64_t now, uint64_t intervalMs);
+
+    // Synchronous firmware command. `buf` carries the request (for a query,
+    // its first bytes, e.g. an iovar name) and receives up to `len` bytes of
+    // answer. Returns false on a transport failure, a timeout, or a negative
+    // firmware status (*fwStatus, when given, receives it).
+    bool BrcmfIoctl(uint32_t cmd, bool set, void* buf, uint32_t len, int32_t* fwStatus = nullptr);
+
+    bool BrcmfSetInt(uint32_t cmd, uint32_t value);
+    bool BrcmfGetInt(uint32_t cmd, uint32_t& value);
+    bool BrcmfIovarSet(const char* name, const void* data, uint32_t len, bool quiet = false);
+    bool BrcmfIovarGet(const char* name, void* out, uint32_t len, bool quiet = false);
+    bool BrcmfIovarSetInt(const char* name, uint32_t value, bool quiet = false);
+    bool BrcmfIovarGetInt(const char* name, uint32_t& value, bool quiet = false);
+
+    // Queue an Ethernet frame for transmission (802.3, 14-byte header).
+    bool BrcmfTxEthernet(const uint8_t* frame, uint32_t len);
+    bool BrcmfTxPending();
+
+    // =========================================================================
+    // BrcmfConnect.cpp -- called from the protocol layer
+    // =========================================================================
+
+    // A firmware event. Runs inside the pump: no ioctls.
+    void BrcmfOnEvent(uint32_t type, uint32_t status, uint32_t reason, uint16_t flags,
+                      const uint8_t* addr, const uint8_t* data, uint32_t len);
+
+    // A received Ethernet frame. Runs inside the pump.
+    void BrcmfOnRxEthernet(const uint8_t* frame, uint32_t len);
+
+    // The firmware halted or stopped answering.
+    void BrcmfOnFirmwareFailure(const char* why);
+
+    // =========================================================================
+    // BrcmfConnect.cpp -- the facade's backend (Wifi.cpp)
+    // =========================================================================
+
+    bool BrcmfProbe(uint64_t arasanMmio);
+    bool BrcmfCompleteInit();
+    void BrcmfShutdown();             // for recovery: power the radio off
+
+    BrcmfState BrcmfGetState();
+    bool BrcmfHasWork(uint64_t now);
+    void BrcmfService();              // pump + connection service
+
+    bool BrcmfStartScan();
+    void BrcmfAbortScan();
+    bool BrcmfScanActive();
+
+    bool BrcmfConnectStart(const uint8_t* bssid, uint8_t channel, const char* ssid,
+                           const char* password, const uint8_t* rsnIe, uint8_t rsnIeLen);
+    void BrcmfConnectAbort();
+    int  BrcmfConnectState();         // IwxConnStateId values
+    bool BrcmfConnectRefusedForSecurity();
+    bool BrcmfJoinFailedOnAuth();
+    const char* BrcmfConnectSsid();
+    const uint8_t* BrcmfConnectBssid();
+    uint8_t BrcmfConnectChannel();
+    bool BrcmfLinkUp();
+    bool BrcmfSendEthernet(const uint8_t* frame, uint32_t len);
+
+    bool BrcmfWpaTxEapol(const uint8_t* body, uint32_t len);
+    bool BrcmfWpaInstallPtk(const uint8_t* tk, uint32_t tkLen, uint8_t cipher);
+    bool BrcmfWpaInstallGtk(const uint8_t* gtk, uint32_t gtkLen, uint8_t keyIdx,
+                            uint8_t cipher, const uint8_t* rsc);
+
+    // =========================================================================
+    // Wifi.cpp -- the facade, called by the backend
+    // =========================================================================
+
+    // One BSS from a scan: its information elements, as in a beacon body
+    // after the fixed fields.
+    void WifiRxBssInfo(const uint8_t* bssid, uint16_t beaconInterval, uint16_t capability,
+                       const uint8_t* ies, uint32_t iesLen, uint8_t channel, int8_t rssiDbm);
+    void WifiRxEthernet(const uint8_t* frame, uint32_t len);
+
+}

kernel/src/Drivers/Net/Wifi/BrcmfChip.cpp

@@ -0,0 +1,849 @@
+/*
+    * BrcmfChip.cpp
+    * Broadcom FullMAC over SDIO: backplane access, core enumeration, clocks,
+    * firmware and NVRAM download, function 2 bring-up, firmware console
+    * Derived in part from Linux's brcmfmac, Copyright (c) 2010-2014 Broadcom
+    * Corporation, ISC license (notice in programs/data/licenses/NOTICES.txt).
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Brcmf.hpp"
+#include "BrcmfSdio.hpp"
+#include <Arch/Cpu.hpp>
+#include <CppLib/Stream.hpp>
+#include <Drivers/Platform/RpiFirmware.hpp>
+#include <Drivers/Platform/RpiGpio.hpp>
+#include <Fs/Vfs.hpp>
+#include <Libraries/Memory.hpp>
+#include <Memory/Heap.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// The radio is a small SoC of its own (a Broadcom "AI" interconnect): an
+// ARM Cortex-R4 with its tightly coupled RAM, the 802.11 MAC/PHY core (D11),
+// ChipCommon (with the PMU) and an SDIO device core. Function 1 of the SDIO
+// card is a window onto that backplane: three window registers select the
+// 32 KiB page, and CMD53 at offset | 0x8000 reaches it.
+//
+// Bring-up follows Linux's brcmfmac (sdio.c, chip.c, firmware.c):
+//   1. route GPIO 34-39 to the Arasan controller, pulse WL_ON (line 1 of the
+//      VideoCore's GPIO expander), enumerate the card;
+//   2. request the ALP clock, read the chip ID, walk the enumeration ROM for
+//      the cores;
+//   3. halt the ARM and hold the D11 core in reset, size the RAM from the
+//      CR4's bank registers;
+//   4. write the firmware at the RAM base and the NVRAM at its top (with the
+//      length token in the last word), write the reset vector to address 0
+//      and release the ARM;
+//   5. wait for the HT clock the firmware brings up, tell the SDIO core the
+//      protocol version, enable function 2 (frames) and its interrupts.
+
+using namespace Kt;
+
+namespace Drivers::Net::Wifi {
+
+    namespace RpiFirmware = Drivers::Platform::RpiFirmware;
+    namespace RpiGpio = Drivers::Platform::RpiGpio;
+
+    BrcmfDevice g_brcmf;
+
+    // =========================================================================
+    // Registers
+    // =========================================================================
+
+    // Function 1, SDIO core local space
+    static constexpr uint32_t F1_WATERMARK   = 0x10008;
+    static constexpr uint32_t F1_DEVICE_CTL  = 0x10009;
+    static constexpr uint32_t F1_SBADDRLOW   = 0x1000A;
+    static constexpr uint32_t F1_SBADDRMID   = 0x1000B;
+    static constexpr uint32_t F1_SBADDRHIGH  = 0x1000C;
+    static constexpr uint32_t F1_CHIPCLKCSR  = 0x1000E;
+    static constexpr uint32_t F1_SDIOPULLUP  = 0x1000F;
+    static constexpr uint32_t F1_MESBUSYCTRL = 0x1001D;
+    static constexpr uint32_t F1_WAKEUPCTRL  = 0x1001E;
+
+    static constexpr uint8_t CLK_FORCE_ALP       = 0x01;
+    static constexpr uint8_t CLK_FORCE_HT        = 0x02;
+    static constexpr uint8_t CLK_ALP_AVAIL_REQ   = 0x08;
+    static constexpr uint8_t CLK_HT_AVAIL_REQ    = 0x10;
+    static constexpr uint8_t CLK_FORCE_HW_REQ_OFF= 0x20;
+    static constexpr uint8_t CLK_ALP_AVAIL       = 0x40;
+    static constexpr uint8_t CLK_HT_AVAIL        = 0x80;
+
+    static constexpr uint8_t DEVCTL_F2WM_ENAB    = 0x10;
+    static constexpr uint8_t MESBUSY_ENAB        = 0x80;
+    static constexpr uint8_t WAKEUP_HTWAIT       = 0x02;
+
+    // CYW43455 values (Linux CY_43455_*): a deeper F2 watermark and
+    // MES busy signalling keep the card's FIFO from overrunning.
+    static constexpr uint8_t F2_WATERMARK_43455  = 0x60;
+    static constexpr uint8_t MES_WATERMARK_43455 = 0x50;
+
+    // Broadcom vendor CCCR
+    static constexpr uint32_t CCCR_BRCM_CARDCAP  = 0xF0;
+    static constexpr uint8_t  CARDCAP_CMD14      = 0x02 | 0x04;   // support | ext
+    static constexpr uint32_t CCCR_BRCM_CARDCTRL = 0xF1;
+    static constexpr uint8_t  CARDCTRL_WLANRESET = 0x02;
+
+    // Backplane
+    static constexpr uint32_t SI_ENUM_BASE = 0x18000000;   // ChipCommon
+    static constexpr uint32_t CC_CHIPID           = 0x000;
+    static constexpr uint32_t CC_CAPS_EXT         = 0x0AC;
+    static constexpr uint32_t CAPS_EXT_AOB        = 0x40;     // PMU in its own "always on" core
+    static constexpr uint32_t CC_EROMPTR          = 0x0FC;
+    static constexpr uint32_t CC_PMUCONTROL       = 0x600;
+    static constexpr uint32_t CC_CHIPCONTROL_ADDR = 0x650;
+    static constexpr uint32_t CC_CHIPCONTROL_DATA = 0x654;
+    static constexpr uint32_t PMU_CTL_RES_RELOAD  = 2u << 13;
+
+    // SDIO device core
+    static constexpr uint32_t SD_INTSTATUS        = 0x20;
+    static constexpr uint32_t SD_HOSTINTMASK      = 0x24;
+    static constexpr uint32_t SD_TOSBMAILBOXDATA  = 0x48;
+    static constexpr uint32_t SDPCM_PROT_VERSION  = 4;
+    static constexpr uint32_t HOSTINTMASK         = 0x000000F0 | (1u << 29);  // HMB SW | CHIPACTIVE
+
+    // Core wrapper (AI)
+    static constexpr uint32_t WRAP_IOCTL      = 0x408;
+    static constexpr uint32_t WRAP_RESET_CTL  = 0x800;
+    static constexpr uint32_t IOCTL_CLK       = 0x0001;
+    static constexpr uint32_t IOCTL_FGC       = 0x0002;
+    static constexpr uint32_t RESET_CTL_RESET = 0x0001;
+
+    static constexpr uint32_t CR4_IOCTL_CPUHALT  = 0x0020;
+    static constexpr uint32_t D11_IOCTL_PHYCLKEN = 0x0004;
+    static constexpr uint32_t D11_IOCTL_PHYRESET = 0x0008;
+
+    // ARM CR4 core
+    static constexpr uint32_t CR4_CAP      = 0x04;
+    static constexpr uint32_t CR4_BANKIDX  = 0x40;
+    static constexpr uint32_t CR4_BANKINFO = 0x44;
+
+    // Core IDs
+    static constexpr uint16_t CORE_CHIPCOMMON = 0x800;
+    static constexpr uint16_t CORE_80211      = 0x812;
+    static constexpr uint16_t CORE_PMU        = 0x827;
+    static constexpr uint16_t CORE_SDIO_DEV   = 0x829;
+    static constexpr uint16_t CORE_ARM_CR4    = 0x83E;
+    static constexpr uint16_t CORE_GCI        = 0x840;
+
+    static constexpr uint16_t CHIP_4345 = 0x4345;
+    static constexpr uint32_t RAMBASE_4345 = 0x198000;
+
+    // =========================================================================
+    // Function 1 and the backplane
+    // =========================================================================
+
+    bool BrcmfF1Read8(uint32_t reg, uint8_t& value) { return Sdio::Read8(Sdio::FN1, reg, value); }
+    bool BrcmfF1Write8(uint32_t reg, uint8_t value) { return Sdio::Write8(Sdio::FN1, reg, value); }
+
+    static bool SetWindow(uint32_t addr) {
+        uint32_t window = addr & ~0x7FFFu;
+        if (window == g_brcmf.Window) return true;
+        bool ok = BrcmfF1Write8(F1_SBADDRLOW, (uint8_t)((window >> 8) & 0x80))
+               && BrcmfF1Write8(F1_SBADDRMID, (uint8_t)(window >> 16))
+               && BrcmfF1Write8(F1_SBADDRHIGH, (uint8_t)(window >> 24));
+        g_brcmf.Window = ok ? window : 0xFFFFFFFF;
+        return ok;
+    }
+
+    bool BrcmfBpSelectWindow(uint32_t addr) { return SetWindow(addr); }
+
+    // 0x8000 in the function 1 address selects a 4-byte backplane access.
+    static constexpr uint32_t SB_ACCESS_4B = 0x8000;
+
+    bool BrcmfBpRead32(uint32_t addr, uint32_t& value) {
+        uint32_t word = 0;
+        if (!SetWindow(addr)) return false;
+        if (!Sdio::ReadBlock(Sdio::FN1, (addr & 0x7FFF) | SB_ACCESS_4B, true, (uint8_t*)&word, 4))
+            return false;
+        value = word;
+        return true;
+    }
+
+    bool BrcmfBpWrite32(uint32_t addr, uint32_t value) {
+        if (!SetWindow(addr)) return false;
+        return Sdio::WriteBlock(Sdio::FN1, (addr & 0x7FFF) | SB_ACCESS_4B, true, (const uint8_t*)&value, 4);
+    }
+
+    static bool BpMem(bool write, uint32_t addr, uint8_t* buffer, uint32_t len) {
+        while (len > 0) {
+            uint32_t offset = addr & 0x7FFF;
+            uint32_t run = 0x8000 - offset;
+            if (run > len) run = len;
+            if (!SetWindow(addr)) return false;
+            bool ok = write ? Sdio::WriteBlock(Sdio::FN1, offset, true, buffer, run)
+                            : Sdio::ReadBlock(Sdio::FN1, offset, true, buffer, run);
+            if (!ok) return false;
+            addr += run;
+            buffer += run;
+            len -= run;
+        }
+        return true;
+    }
+
+    bool BrcmfBpReadMem(uint32_t addr, uint8_t* buffer, uint32_t len) {
+        return BpMem(false, addr, buffer, len);
+    }
+
+    bool BrcmfBpWriteMem(uint32_t addr, const uint8_t* buffer, uint32_t len) {
+        return BpMem(true, addr, (uint8_t*)buffer, len);
+    }
+
+    static uint32_t Rd(uint32_t addr) {
+        uint32_t v = 0;
+        BrcmfBpRead32(addr, v);
+        return v;
+    }
+
+    // =========================================================================
+    // Cores
+    // =========================================================================
+
+    static const BrcmfCore* FindCore(uint16_t id) {
+        for (uint32_t i = 0; i < g_brcmf.CoreCount; i++)
+            if (g_brcmf.Cores[i].Id == id) return &g_brcmf.Cores[i];
+        return nullptr;
+    }
+
+    // Enumeration ROM (Linux brcmf_chip_dmp_erom_scan).
+    static constexpr uint32_t DESC_TYPE_MSK       = 0x0000000F;
+    static constexpr uint32_t DESC_VALID          = 0x00000001;
+    static constexpr uint32_t DESC_COMPONENT      = 0x00000001;
+    static constexpr uint32_t DESC_MASTER_PORT    = 0x00000003;
+    static constexpr uint32_t DESC_ADDRESS        = 0x00000005;
+    static constexpr uint32_t DESC_ADDRSIZE_GT32  = 0x00000008;
+    static constexpr uint32_t DESC_EOT            = 0x0000000F;
+    static constexpr uint32_t COMP_PARTNUM_MASK   = 0x000FFF00, COMP_PARTNUM_SHIFT = 8;
+    static constexpr uint32_t COMP_REV_MASK       = 0xFF000000, COMP_REV_SHIFT = 24;
+    static constexpr uint32_t COMP_NSW_MASK       = 0x00F80000, COMP_NSW_SHIFT = 19;
+    static constexpr uint32_t COMP_NMW_MASK       = 0x0007C000, COMP_NMW_SHIFT = 14;
+    static constexpr uint32_t SLAVE_ADDR_BASE     = 0xFFFFF000;
+    static constexpr uint32_t SLAVE_TYPE_MASK     = 0x000000C0, SLAVE_TYPE_SHIFT = 6;
+    static constexpr uint32_t SLAVE_TYPE_SLAVE    = 0;
+    static constexpr uint32_t SLAVE_TYPE_SWRAP    = 2;
+    static constexpr uint32_t SLAVE_TYPE_MWRAP    = 3;
+    static constexpr uint32_t SLAVE_SIZE_MASK     = 0x00000030, SLAVE_SIZE_SHIFT = 4;
+    static constexpr uint32_t SLAVE_SIZE_4K       = 0;
+    static constexpr uint32_t SLAVE_SIZE_8K       = 1;
+    static constexpr uint32_t SLAVE_SIZE_DESC     = 3;
+
+    static uint32_t EromNext(uint32_t& erom, uint32_t* type) {
+        uint32_t v = Rd(erom);
+        erom += 4;
+        uint32_t t = v & DESC_TYPE_MSK;
+        if ((t & ~DESC_ADDRSIZE_GT32) == DESC_ADDRESS) t = DESC_ADDRESS;
+        if (type) *type = t;
+        return v;
+    }
+
+    static bool EromRegAddr(uint32_t& erom, uint32_t& regBase, uint32_t& wrapBase) {
+        regBase = 0;
+        wrapBase = 0;
+        uint32_t desc = 0, val = 0;
+        uint32_t wrapType;
+
+        // One descriptor decides it, as in Linux: a master port means the
+        // core's wrapper is a master wrapper (any further master ports are
+        // skipped by the address search below); an address straight away
+        // means a slave wrapper. Cores with master ports -- the CR4, D11,
+        // the SDIO core, and ChipCommon on these chips -- must not be
+        // looked up as slave-wrapped: their wrapper is never found, and the
+        // halt/release writes go nowhere.
+        val = EromNext(erom, &desc);
+        if (desc == DESC_MASTER_PORT) {
+            wrapType = SLAVE_TYPE_MWRAP;
+        } else if (desc == DESC_ADDRESS) {
+            erom -= 4;
+            wrapType = SLAVE_TYPE_SWRAP;
+        } else {
+            erom -= 4;
+            return false;
+        }
+
+        for (int guard = 0; guard < 64; guard++) {
+            do {
+                val = EromNext(erom, &desc);
+                if (desc == DESC_EOT) { erom -= 4; return false; }
+            } while (desc != DESC_ADDRESS && desc != DESC_COMPONENT);
+
+            if (desc == DESC_COMPONENT) { erom -= 4; return true; }
+
+            if (val & DESC_ADDRSIZE_GT32) EromNext(erom, nullptr);
+            uint32_t sizeType = (val & SLAVE_SIZE_MASK) >> SLAVE_SIZE_SHIFT;
+            if (sizeType == SLAVE_SIZE_DESC) {
+                uint32_t size = EromNext(erom, nullptr);
+                if (size & DESC_ADDRSIZE_GT32) EromNext(erom, nullptr);
+            }
+            if (sizeType != SLAVE_SIZE_4K && sizeType != SLAVE_SIZE_8K) continue;
+
+            uint32_t slaveType = (val & SLAVE_TYPE_MASK) >> SLAVE_TYPE_SHIFT;
+            if (regBase == 0 && slaveType == SLAVE_TYPE_SLAVE) regBase = val & SLAVE_ADDR_BASE;
+            if (wrapBase == 0 && slaveType == wrapType) wrapBase = val & SLAVE_ADDR_BASE;
+            if (regBase && wrapBase) return true;
+        }
+        return false;
+    }
+
+    static bool ScanCores() {
+        g_brcmf.CoreCount = 0;
+        uint32_t erom = Rd(SI_ENUM_BASE + CC_EROMPTR);
+        if (erom == 0 || erom == 0xFFFFFFFF) return false;
+
+        uint32_t desc = 0;
+        for (int guard = 0; guard < 512 && desc != DESC_EOT; guard++) {
+            uint32_t val = EromNext(erom, &desc);
+            if (!(val & DESC_VALID) || desc != DESC_COMPONENT) continue;
+            uint16_t id = (uint16_t)((val & COMP_PARTNUM_MASK) >> COMP_PARTNUM_SHIFT);
+
+            val = EromNext(erom, &desc);
+            if (desc != DESC_COMPONENT) return false;
+            uint32_t nmw = (val & COMP_NMW_MASK) >> COMP_NMW_SHIFT;
+            uint32_t nsw = (val & COMP_NSW_MASK) >> COMP_NSW_SHIFT;
+            uint8_t rev = (uint8_t)((val & COMP_REV_MASK) >> COMP_REV_SHIFT);
+            if (nmw + nsw == 0 && id != CORE_PMU && id != CORE_GCI) continue;
+
+            uint32_t base = 0, wrap = 0;
+            if (!EromRegAddr(erom, base, wrap)) continue;
+            if (g_brcmf.CoreCount < sizeof(g_brcmf.Cores) / sizeof(g_brcmf.Cores[0])) {
+                g_brcmf.Cores[g_brcmf.CoreCount++] = { id, rev, base, wrap };
+            }
+        }
+        const BrcmfCore* cr4 = FindCore(CORE_ARM_CR4);
+        return FindCore(CORE_CHIPCOMMON) && FindCore(CORE_SDIO_DEV) && cr4 && cr4->Wrap != 0;
+    }
+
+    static void CoreDisable(const BrcmfCore* core, uint32_t prereset, uint32_t reset) {
+        if (!(Rd(core->Wrap + WRAP_RESET_CTL) & RESET_CTL_RESET)) {
+            BrcmfBpWrite32(core->Wrap + WRAP_IOCTL, prereset | IOCTL_FGC | IOCTL_CLK);
+            Rd(core->Wrap + WRAP_IOCTL);
+            BrcmfBpWrite32(core->Wrap + WRAP_RESET_CTL, RESET_CTL_RESET);
+            Timekeeping::BusyWaitUs(20);
+            for (int i = 0; i < 30 && Rd(core->Wrap + WRAP_RESET_CTL) != RESET_CTL_RESET; i++)
+                Timekeeping::BusyWaitUs(10);
+        }
+        BrcmfBpWrite32(core->Wrap + WRAP_IOCTL, reset | IOCTL_FGC | IOCTL_CLK);
+        Rd(core->Wrap + WRAP_IOCTL);
+    }
+
+    static void CoreReset(const BrcmfCore* core, uint32_t prereset, uint32_t reset, uint32_t postreset) {
+        CoreDisable(core, prereset, reset);
+        for (int i = 0; i < 50 && (Rd(core->Wrap + WRAP_RESET_CTL) & RESET_CTL_RESET); i++) {
+            BrcmfBpWrite32(core->Wrap + WRAP_RESET_CTL, 0);
+            Timekeeping::BusyWaitUs(50);
+        }
+        BrcmfBpWrite32(core->Wrap + WRAP_IOCTL, postreset | IOCTL_CLK);
+        Rd(core->Wrap + WRAP_IOCTL);
+    }
+
+    // The CR4's TCM: ATCM and BTCM bank counts in CAP, each bank's size in
+    // 8 KiB units in BANKINFO.
+    static uint32_t Cr4RamSize(const BrcmfCore* cr4) {
+        uint32_t cap = Rd(cr4->Base + CR4_CAP);
+        uint32_t banks = (cap & 0xF) + ((cap >> 4) & 0xF);
+        uint32_t size = 0;
+        for (uint32_t i = 0; i < banks; i++) {
+            BrcmfBpWrite32(cr4->Base + CR4_BANKIDX, i);
+            uint32_t info = Rd(cr4->Base + CR4_BANKINFO);
+            size += ((info & 0x3F) + 1) * 8192;
+        }
+        return size;
+    }
+
+    // =========================================================================
+    // Firmware files
+    // =========================================================================
+
+    uint8_t* BrcmfLoadFile(const char* ext, uint32_t& size, bool required) {
+        char path[80];
+        int n = 0;
+        auto add = [&](const char* s) { while (*s && n < 78) path[n++] = *s++; };
+        add("0:/os/firmware/brcm/brcmfmac");
+        add(g_brcmf.FwName);
+        add("-sdio.");
+        add(ext);
+        path[n] = '\0';
+
+        size = 0;
+        Fs::Vfs::BackendFile file;
+        if (Fs::Vfs::OpenBackendFile(path, file) < 0) {
+            if (required) KernelLogStream(ERROR, "WiFi") << "Firmware file not found: " << path;
+            return nullptr;
+        }
+        uint64_t fileSize = Fs::Vfs::GetBackendFileSize(file);
+        if (fileSize == 0 || fileSize > 4 * 1024 * 1024) {
+            Fs::Vfs::CloseBackendFile(file);
+            KernelLogStream(ERROR, "WiFi") << "Unusable firmware file " << path;
+            return nullptr;
+        }
+        // Rounded up and zero padded: everything goes over the bus in dwords.
+        uint32_t padded = ((uint32_t)fileSize + 3) & ~3u;
+        uint8_t* data = (uint8_t*)Memory::g_heap->Request(padded);
+        if (!data) {
+            Fs::Vfs::CloseBackendFile(file);
+            return nullptr;
+        }
+        memset(data, 0, padded);
+        int got = Fs::Vfs::ReadBackendFile(file, data, 0, fileSize);
+        Fs::Vfs::CloseBackendFile(file);
+        if (got != (int)fileSize) {
+            Memory::g_heap->Free(data);
+            KernelLogStream(ERROR, "WiFi") << "Short read of " << path;
+            return nullptr;
+        }
+        size = (uint32_t)fileSize;
+        KernelLogStream(INFO, "WiFi") << "Loaded " << path << " (" << (uint64_t)size << " bytes)";
+        return data;
+    }
+
+    // The NVRAM text ("key=value" lines, '#' comments) becomes the binary
+    // form the firmware reads: NUL-terminated pairs, zero padded to a dword
+    // (at least one extra NUL), then a token word holding the padded length
+    // in dwords and its complement (Linux brcmf_fw_nvram_strip).
+    static uint8_t* BuildNvram(const uint8_t* text, uint32_t len, uint32_t& outLen) {
+        uint8_t* out = (uint8_t*)Memory::g_heap->Request(len + 8);
+        if (!out) return nullptr;
+        uint32_t n = 0;
+        uint32_t i = 0;
+        auto printable = [](uint8_t c) { return c > 0x20 && c < 0x7F && c != '#'; };
+        while (i < len) {
+            while (i < len && (text[i] == ' ' || text[i] == '\t' || text[i] == '\r' || text[i] == '\n')) i++;
+            if (i >= len) break;
+            if (text[i] == '#') {
+                while (i < len && text[i] != '\n') i++;
+                continue;
+            }
+            uint32_t start = i;
+            bool sawEquals = false;
+            while (i < len && printable(text[i])) {
+                if (text[i] == '=') sawEquals = true;
+                i++;
+            }
+            if (sawEquals && i > start) {
+                memcpy(out + n, text + start, i - start);
+                n += i - start;
+                out[n++] = '\0';
+            }
+            while (i < len && text[i] != '\n') i++;
+        }
+        uint32_t padded = (n + 1 + 3) & ~3u;
+        while (n < padded) out[n++] = 0;
+        uint32_t words = padded / 4;
+        uint32_t token = (~words << 16) | (words & 0xFFFF);
+        memcpy(out + n, &token, 4);
+        outLen = n + 4;
+        return out;
+    }
+
+    static bool DownloadAndVerify(uint32_t addr, const uint8_t* data, uint32_t len) {
+        if (!BrcmfBpWriteMem(addr, data, len)) return false;
+
+        // Read the image back: a marginal SDIO bus shows up here as a clean
+        // error instead of as a firmware that never starts.
+        uint8_t* check = (uint8_t*)Memory::g_heap->Request(4096);
+        if (!check) return true;
+        bool ok = true;
+        for (uint32_t off = 0; off < len && ok; off += 4096) {
+            uint32_t run = len - off < 4096 ? len - off : 4096;
+            if (!BrcmfBpReadMem(addr + off, check, run) || memcmp(check, data + off, run) != 0) {
+                KernelLogStream(ERROR, "WiFi") << "Firmware verify failed at offset 0x"
+                    << base::hex << (uint64_t)off << base::dec;
+                ok = false;
+            }
+        }
+        Memory::g_heap->Free(check);
+        return ok;
+    }
+
+    // =========================================================================
+    // Bring-up
+    // =========================================================================
+
+    static bool WaitClock(uint8_t bit, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            uint8_t v = 0;
+            if (BrcmfF1Read8(F1_CHIPCLKCSR, v) && (v & bit)) return true;
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
+            Timekeeping::BusyWaitUs(200);
+        }
+    }
+
+    void BrcmfChipPowerOff() {
+        RpiFirmware::SetGpioOutput(RpiFirmware::EXPGPIO_WL_ON, false);
+    }
+
+    static bool PowerOnAndEnumerate() {
+        // GPIO 34-39: CLK, CMD, DAT0-3 on ALT3 (the Arasan), pulled up but
+        // for the clock, as Linux's sdio_pins.
+        for (uint32_t pin = 34; pin <= 39; pin++) {
+            RpiGpio::SetFunction(pin, RpiGpio::Function::Alt3);
+            RpiGpio::SetPull(pin, pin == 34 ? RpiGpio::Pull::None : RpiGpio::Pull::Up);
+        }
+
+        if (!RpiFirmware::SetGpioOutput(RpiFirmware::EXPGPIO_WL_ON, false)) {
+            KernelLogStream(ERROR, "WiFi") << "Cannot switch the radio's power (WL_ON)";
+            return false;
+        }
+        Timekeeping::BusyWaitMs(20);
+        RpiFirmware::SetGpioOutput(RpiFirmware::EXPGPIO_WL_ON, true);
+        Timekeeping::BusyWaitMs(250);
+
+        // The legacy EMMC clock feeds the Arasan; nothing on the boot path
+        // uses that controller, so make sure the clock is not gated.
+        if (!RpiFirmware::SetClockState(RpiFirmware::CLOCK_EMMC, true))
+            KernelLogStream(WARNING, "WiFi") << "Could not switch the EMMC clock on";
+        uint32_t clockHz = 0;
+        if (!RpiFirmware::GetClockRate(RpiFirmware::CLOCK_EMMC, clockHz)) clockHz = 0;
+        if (!Sdio::HostInit(g_brcmf.ArasanMmio, clockHz)) return false;
+        return Sdio::CardInit();
+    }
+
+    static bool AttachChip() {
+        g_brcmf.Window = 0xFFFFFFFF;
+
+        // ALP first: the backplane runs from it until the firmware has set
+        // up the PLL.
+        uint8_t req = CLK_FORCE_HW_REQ_OFF | CLK_ALP_AVAIL_REQ;
+        uint8_t v = 0;
+        if (!BrcmfF1Write8(F1_CHIPCLKCSR, req) || !BrcmfF1Read8(F1_CHIPCLKCSR, v) ||
+            (v & ~(CLK_ALP_AVAIL | CLK_HT_AVAIL)) != req) {
+            KernelLogStream(ERROR, "WiFi") << "ChipClkCSR: wrote 0x" << base::hex << (uint64_t)req
+                << ", read 0x" << (uint64_t)v << base::dec;
+            return false;
+        }
+        if (!WaitClock(CLK_ALP_AVAIL, 100)) {
+            KernelLogStream(ERROR, "WiFi") << "ALP clock never became available";
+            return false;
+        }
+        BrcmfF1Write8(F1_CHIPCLKCSR, CLK_FORCE_HW_REQ_OFF | CLK_FORCE_ALP);
+        Timekeeping::BusyWaitUs(65);
+        BrcmfF1Write8(F1_SDIOPULLUP, 0);
+
+        uint32_t chipId = 0;
+        if (!BrcmfBpRead32(SI_ENUM_BASE + CC_CHIPID, chipId)) {
+            KernelLogStream(ERROR, "WiFi") << "Backplane read failed";
+            return false;
+        }
+        g_brcmf.ChipId = (uint16_t)(chipId & 0xFFFF);
+        g_brcmf.ChipRev = (uint8_t)((chipId >> 16) & 0xF);
+        uint32_t socType = chipId >> 28;
+
+        KernelLogStream(INFO, "WiFi") << "Broadcom chip " << base::hex << (uint64_t)g_brcmf.ChipId
+            << base::dec << " rev " << (uint64_t)g_brcmf.ChipRev;
+
+        if (g_brcmf.ChipId != CHIP_4345 || socType != 1) {
+            KernelLogStream(ERROR, "WiFi") << "Unsupported chip (only the BCM4345 family: "
+                "CYW43455 / CYW43456)";
+            return false;
+        }
+        // Linux's table: revision 9 is the CYW43456 (Pi 400), the others
+        // the CYW43455 (Pi 4B).
+        g_brcmf.FwName = g_brcmf.ChipRev == 9 ? "43456" : "43455";
+
+        if (!ScanCores()) {
+            KernelLogStream(ERROR, "WiFi") << "Enumeration ROM scan found no usable cores";
+            return false;
+        }
+        // Linux brcmf_chip_get_pmu: the PMU registers live in a core of their
+        // own only when ChipCommon (rev 35+) says the always-on block exists;
+        // otherwise they are ChipCommon's, at the same offsets.
+        const BrcmfCore* cc = FindCore(CORE_CHIPCOMMON);
+        const BrcmfCore* pmu = FindCore(CORE_PMU);
+        uint32_t capsExt = cc->Rev >= 35 ? Rd(cc->Base + CC_CAPS_EXT) : 0;
+        g_brcmf.PmuBase = (pmu && (capsExt & CAPS_EXT_AOB)) ? pmu->Base : cc->Base;
+        g_brcmf.SdioCoreBase = FindCore(CORE_SDIO_DEV)->Base;
+
+        for (uint32_t i = 0; i < g_brcmf.CoreCount; i++) {
+            const BrcmfCore& c = g_brcmf.Cores[i];
+            KernelLogStream(INFO, "WiFi") << "  core 0x" << base::hex << (uint64_t)c.Id << " rev "
+                << base::dec << (uint64_t)c.Rev << " at 0x" << base::hex << (uint64_t)c.Base
+                << ", wrapper 0x" << (uint64_t)c.Wrap << base::dec;
+        }
+        KernelLogStream(INFO, "WiFi") << "ChipCommon rev " << (uint64_t)cc->Rev << ", caps_ext 0x"
+            << base::hex << (uint64_t)capsExt << ", PMU registers at 0x" << (uint64_t)g_brcmf.PmuBase
+            << base::dec;
+
+        // Passive: ARM halted, D11 held in reset with its PHY clock on.
+        const BrcmfCore* cr4 = FindCore(CORE_ARM_CR4);
+        CoreReset(cr4, CR4_IOCTL_CPUHALT, CR4_IOCTL_CPUHALT, CR4_IOCTL_CPUHALT);
+        if (const BrcmfCore* d11 = FindCore(CORE_80211)) {
+            CoreReset(d11, D11_IOCTL_PHYRESET | D11_IOCTL_PHYCLKEN, D11_IOCTL_PHYCLKEN, D11_IOCTL_PHYCLKEN);
+        }
+        uint32_t haltIoctl = Rd(cr4->Wrap + WRAP_IOCTL);
+        uint32_t haltReset = Rd(cr4->Wrap + WRAP_RESET_CTL);
+        if (haltIoctl != (CR4_IOCTL_CPUHALT | IOCTL_CLK) || (haltReset & RESET_CTL_RESET)) {
+            KernelLogStream(ERROR, "WiFi") << "The CR4 did not take the halt (wrapper 0x" << base::hex
+                << (uint64_t)cr4->Wrap << ": ioctl 0x" << (uint64_t)haltIoctl << " reset 0x"
+                << (uint64_t)haltReset << base::dec << ", expected 0x21 / 0x0)";
+            return false;
+        }
+
+        g_brcmf.RamBase = RAMBASE_4345;
+        g_brcmf.RamSize = Cr4RamSize(cr4);
+        if (g_brcmf.RamSize == 0 || g_brcmf.RamSize > 4 * 1024 * 1024) {
+            KernelLogStream(ERROR, "WiFi") << "Implausible RAM size " << (uint64_t)g_brcmf.RamSize;
+            return false;
+        }
+
+        // An SDIO card reset also resets the WLAN backplane, and a backplane
+        // reset reloads the PMU state.
+        uint8_t cardCtrl = 0;
+        if (Sdio::Read8(Sdio::FN0, CCCR_BRCM_CARDCTRL, cardCtrl))
+            Sdio::Write8(Sdio::FN0, CCCR_BRCM_CARDCTRL, cardCtrl | CARDCTRL_WLANRESET);
+        BrcmfBpWrite32(g_brcmf.PmuBase + CC_PMUCONTROL,
+                       Rd(g_brcmf.PmuBase + CC_PMUCONTROL) | PMU_CTL_RES_RELOAD);
+
+        KernelLogStream(INFO, "WiFi") << "CYW" << g_brcmf.FwName << ": " << (uint64_t)g_brcmf.CoreCount
+            << " cores, " << (uint64_t)(g_brcmf.RamSize / 1024) << " KiB RAM at 0x" << base::hex
+            << (uint64_t)g_brcmf.RamBase << base::dec;
+        return true;
+    }
+
+    static bool DownloadFirmware() {
+        uint32_t fwLen = 0, nvLen = 0;
+        uint8_t* fw = BrcmfLoadFile("bin", fwLen, true);
+        if (!fw) return false;
+        uint8_t* nvText = BrcmfLoadFile("txt", nvLen, true);
+        if (!nvText) {
+            Memory::g_heap->Free(fw);
+            return false;
+        }
+
+        uint32_t nvramLen = 0;
+        uint8_t* nvram = BuildNvram(nvText, nvLen, nvramLen);
+        Memory::g_heap->Free(nvText);
+        uint32_t fwPadded = (fwLen + 3) & ~3u;
+
+        bool ok = nvram && fwPadded + nvramLen < g_brcmf.RamSize;
+        if (!ok) KernelLogStream(ERROR, "WiFi") << "Firmware and NVRAM do not fit in the radio's RAM";
+
+        uint32_t rstvec = 0;
+        memcpy(&rstvec, fw, 4);
+
+        // Download under a plain ALP request, as Linux does (alp_only):
+        // the probe's FORCE_HW_REQ_OFF | FORCE_ALP must be gone before the
+        // ARM starts, or the firmware's own clock requests -- the HT clock
+        // its PLL setup asks for above all -- are blocked.
+        if (ok) {
+            BrcmfF1Write8(F1_CHIPCLKCSR, CLK_ALP_AVAIL_REQ);
+            if (!WaitClock(CLK_ALP_AVAIL, 100))
+                KernelLogStream(WARNING, "WiFi") << "ALP clock lost on its plain request";
+        }
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        if (ok && !DownloadAndVerify(g_brcmf.RamBase, fw, fwPadded)) {
+            KernelLogStream(ERROR, "WiFi") << "Firmware download failed";
+            ok = false;
+        }
+        if (ok && !DownloadAndVerify(g_brcmf.RamBase + g_brcmf.RamSize - nvramLen, nvram, nvramLen)) {
+            KernelLogStream(ERROR, "WiFi") << "NVRAM download failed";
+            ok = false;
+        }
+        if (ok) {
+            KernelLogStream(INFO, "WiFi") << "Firmware and NVRAM written in "
+                << (Timekeeping::GetMilliseconds() - start) << " ms";
+        }
+
+        Memory::g_heap->Free(fw);
+        if (nvram) Memory::g_heap->Free(nvram);
+        if (!ok) return false;
+
+        // Start the ARM: clear the SDIO core's interrupts, put the reset
+        // vector at address 0 and take the CPU out of halt.
+        BrcmfBpWrite32(g_brcmf.SdioCoreBase + SD_INTSTATUS, 0xFFFFFFFF);
+        uint32_t vecBack = 0;
+        bool vecOk = BrcmfBpWriteMem(0, (const uint8_t*)&rstvec, 4)
+                  && BrcmfBpReadMem(0, (uint8_t*)&vecBack, 4) && vecBack == rstvec;
+        const BrcmfCore* cr4 = FindCore(CORE_ARM_CR4);
+        CoreReset(cr4, CR4_IOCTL_CPUHALT, 0, 0);
+        if (!vecOk)
+            KernelLogStream(ERROR, "WiFi") << "Reset vector did not stick: address 0 reads 0x"
+                << base::hex << (uint64_t)vecBack << base::dec;
+        KernelLogStream(INFO, "WiFi") << "Reset vector 0x" << base::hex << (uint64_t)rstvec
+            << "; CR4 released: ioctl 0x" << (uint64_t)Rd(cr4->Wrap + WRAP_IOCTL) << " reset 0x"
+            << (uint64_t)Rd(cr4->Wrap + WRAP_RESET_CTL) << base::dec << " (expect 0x1 / 0x0)";
+        return true;
+    }
+
+    // Save/restore: the firmware can power its domains down between
+    // frames; the host then has to allow for the HT clock coming back.
+    static bool SrCapable() {
+        BrcmfBpWrite32(g_brcmf.PmuBase + CC_CHIPCONTROL_ADDR, 3);
+        return (Rd(g_brcmf.PmuBase + CC_CHIPCONTROL_DATA) & (1u << 2)) != 0;
+    }
+
+    static bool StartFrames() {
+        // Clear the ALP force, then ask for HT: the running firmware brings
+        // up the PLL.
+        BrcmfF1Write8(F1_CHIPCLKCSR, 0);
+        BrcmfF1Write8(F1_CHIPCLKCSR, CLK_HT_AVAIL_REQ);
+        bool ht = WaitClock(CLK_HT_AVAIL, 20);
+        if (!ht) {
+            // Not yet: a snapshot 20 ms in tells a CPU that never ran from
+            // one that ran and was reset.
+            const BrcmfCore* cr4 = FindCore(CORE_ARM_CR4);
+            uint8_t csr = 0;
+            BrcmfF1Read8(F1_CHIPCLKCSR, csr);
+            KernelLogStream(INFO, "WiFi") << "20 ms after release: ChipClkCSR 0x" << base::hex
+                << (uint64_t)csr << ", CR4 ioctl 0x" << (uint64_t)Rd(cr4->Wrap + WRAP_IOCTL) << " reset 0x"
+                << (uint64_t)Rd(cr4->Wrap + WRAP_RESET_CTL) << ", RAM top 0x"
+                << (uint64_t)Rd(g_brcmf.RamBase + g_brcmf.RamSize - 4) << base::dec;
+            ht = WaitClock(CLK_HT_AVAIL, 2000);
+        }
+        if (!ht) {
+            // What the hardware was doing, for the next attempt: the clock
+            // register (or whether function 1 still answers at all), the
+            // CPU's wrapper state, and whether the firmware ever replaced
+            // the NVRAM token in the last word of RAM with its shared area.
+            uint8_t csr = 0;
+            bool csrOk = BrcmfF1Read8(F1_CHIPCLKCSR, csr);
+            const BrcmfCore* cr4 = FindCore(CORE_ARM_CR4);
+            uint32_t ioctl = Rd(cr4->Wrap + WRAP_IOCTL);
+            uint32_t reset = Rd(cr4->Wrap + WRAP_RESET_CTL);
+            uint32_t top = Rd(g_brcmf.RamBase + g_brcmf.RamSize - 4);
+            KernelLogStream(ERROR, "WiFi") << "HT clock never became available (firmware did not start)";
+            KernelLogStream(ERROR, "WiFi") << "ChipClkCSR " << (csrOk ? "0x" : "unreadable")
+                << base::hex << (uint64_t)(csrOk ? csr : 0) << ", CR4 ioctl 0x" << (uint64_t)ioctl
+                << " reset 0x" << (uint64_t)reset << ", RAM top word 0x" << (uint64_t)top
+                << base::dec << (((~top >> 16) & 0xFFFF) == (top & 0xFFFF)
+                                 ? " (still the NVRAM token)" : " (written by the firmware)");
+            return false;
+        }
+
+        uint8_t saveClk = 0;
+        BrcmfF1Read8(F1_CHIPCLKCSR, saveClk);
+        BrcmfF1Write8(F1_CHIPCLKCSR, saveClk | CLK_FORCE_HT);
+
+        BrcmfBpWrite32(g_brcmf.SdioCoreBase + SD_TOSBMAILBOXDATA, SDPCM_PROT_VERSION << 16);
+        if (!Sdio::EnableFunction(Sdio::FN2, true, 3000)) {
+            KernelLogStream(ERROR, "WiFi") << "Function 2 did not become ready";
+            Sdio::EnableFunction(Sdio::FN2, false, 0);
+            return false;
+        }
+
+        g_brcmf.HostIntMask = HOSTINTMASK;
+        BrcmfBpWrite32(g_brcmf.SdioCoreBase + SD_HOSTINTMASK, HOSTINTMASK);
+
+        BrcmfF1Write8(F1_WATERMARK, F2_WATERMARK_43455);
+        uint8_t devCtl = 0;
+        if (BrcmfF1Read8(F1_DEVICE_CTL, devCtl))
+            BrcmfF1Write8(F1_DEVICE_CTL, devCtl | DEVCTL_F2WM_ENAB);
+        BrcmfF1Write8(F1_MESBUSYCTRL, MES_WATERMARK_43455 | MESBUSY_ENAB);
+
+        g_brcmf.SrEnabled = SrCapable();
+        if (g_brcmf.SrEnabled) {
+            uint8_t wake = 0;
+            if (BrcmfF1Read8(F1_WAKEUPCTRL, wake)) BrcmfF1Write8(F1_WAKEUPCTRL, wake | WAKEUP_HTWAIT);
+            Sdio::Write8(Sdio::FN0, CCCR_BRCM_CARDCAP, CARDCAP_CMD14);
+            BrcmfF1Write8(F1_CHIPCLKCSR, CLK_FORCE_HT);
+        } else {
+            BrcmfF1Write8(F1_CHIPCLKCSR, saveClk);
+        }
+
+        // Master, function 1 and function 2.
+        Sdio::EnableCardInterrupts(0x07);
+        return true;
+    }
+
+    bool BrcmfChipBringUp() {
+        g_brcmf.BusLock.Acquire();
+        g_brcmf.SharedAddr = 0;
+        g_brcmf.ConsoleIdx = 0;
+        g_brcmf.RamSize = 0;
+        bool ok = PowerOnAndEnumerate() && AttachChip() && DownloadFirmware() && StartFrames();
+        g_brcmf.BusLock.Release();
+        if (!ok) {
+            // A firmware that started and then trapped has said why on its
+            // console (no-op when it never got that far).
+            BrcmfDumpConsole();
+            BrcmfChipPowerOff();
+        }
+        return ok;
+    }
+
+    // =========================================================================
+    // Firmware console and trap report
+    // =========================================================================
+
+    static bool InRam(uint32_t addr, uint32_t len) {
+        return addr >= g_brcmf.RamBase && addr + len <= g_brcmf.RamBase + g_brcmf.RamSize
+            && addr + len >= addr;
+    }
+
+    static bool ReadShared(uint32_t sh[16]) {
+        if (!g_brcmf.SharedAddr) {
+            uint32_t addr = Rd(g_brcmf.RamBase + g_brcmf.RamSize - 4);
+            // Still the NVRAM token: the firmware has not published it.
+            if (((~addr >> 16) & 0xFFFF) == (addr & 0xFFFF) || !InRam(addr, 64)) return false;
+            g_brcmf.SharedAddr = addr;
+        }
+        return BrcmfBpReadMem(g_brcmf.SharedAddr, (uint8_t*)sh, 64);
+    }
+
+    static void EmitConsole(const char* buf, uint32_t size, uint32_t from, uint32_t to) {
+        char line[161];
+        uint32_t n = 0;
+        for (uint32_t i = from; i != to; i = (i + 1) % size) {
+            char c = buf[i];
+            if (c == '\r') continue;
+            if (c == '\n' || n == sizeof(line) - 1) {
+                line[n] = '\0';
+                if (n) KernelLogStream(INFO, "WiFi-FW") << line;
+                n = 0;
+                if (c == '\n') continue;
+            }
+            line[n++] = (c >= 0x20 && c < 0x7F) ? c : '.';
+        }
+        line[n] = '\0';
+        if (n) KernelLogStream(INFO, "WiFi-FW") << line;
+    }
+
+    void BrcmfDumpConsole() {
+        if (g_brcmf.RamSize == 0) return;
+        g_brcmf.BusLock.Acquire();
+
+        // sdpcm_shared: flags, trap_addr, assert_exp_addr, assert_file_addr,
+        // assert_line, console_addr, msgtrace_addr, tag[32], brpt_addr
+        uint32_t sh[16] = {};
+        if (!ReadShared(sh)) {
+            g_brcmf.BusLock.Release();
+            return;
+        }
+        uint32_t flags = sh[0];
+
+        // rte_console: vcons_in, vcons_out, then the log: buf, buf_size,
+        // idx, out_idx.
+        uint32_t log[4] = {};
+        if (InRam(sh[5], 24) && BrcmfBpReadMem(sh[5] + 8, (uint8_t*)log, 16)
+            && log[1] > 0 && log[1] <= 16384 && InRam(log[0], log[1]) && log[2] < log[1]) {
+            uint32_t size = log[1];
+            char* buf = (char*)Memory::g_heap->Request((size + 3) & ~3u);
+            if (buf && BrcmfBpReadMem(log[0], (uint8_t*)buf, (size + 3) & ~3u)) {
+                if (g_brcmf.ConsoleIdx >= size) g_brcmf.ConsoleIdx = 0;
+                EmitConsole(buf, size, g_brcmf.ConsoleIdx, log[2]);
+                g_brcmf.ConsoleIdx = log[2];
+            }
+            if (buf) Memory::g_heap->Free(buf);
+        }
+
+        if ((flags & 0x0400) && InRam(sh[1], 72)) {
+            // brcmf_trap_info: type, epc, cpsr, spsr, r0-r12, sp, lr, pc
+            uint32_t trap[20] = {};
+            if (BrcmfBpReadMem(sh[1], (uint8_t*)trap, sizeof(trap))) {
+                KernelLogStream(ERROR, "WiFi-FW") << "Firmware trap: type 0x" << base::hex
+                    << (uint64_t)trap[0] << ", epc 0x" << (uint64_t)trap[1] << ", lr 0x"
+                    << (uint64_t)trap[18] << ", pc 0x" << (uint64_t)trap[19] << base::dec;
+            }
+        }
+        if (flags & 0x0200) {
+            char file[64] = {};
+            if (InRam(sh[3], 60)) BrcmfBpReadMem(sh[3], (uint8_t*)file, 60);
+            file[63] = '\0';
+            KernelLogStream(ERROR, "WiFi-FW") << "Firmware assert at " << file << ":"
+                << (uint64_t)sh[4];
+        }
+        g_brcmf.BusLock.Release();
+    }
+
+}

kernel/src/Drivers/Net/Wifi/BrcmfConnect.cpp

@@ -0,0 +1,902 @@
+/*
+    * BrcmfConnect.cpp
+    * Broadcom FullMAC over SDIO: firmware preinit (CLM, events), escan,
+    * join, key installation, link state, and the entry points the Wi-Fi
+    * facade calls
+    * Derived in part from Linux's brcmfmac, Copyright (c) 2010-2014 Broadcom
+    * Corporation, ISC license (notice in programs/data/licenses/NOTICES.txt).
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Brcmf.hpp"
+#include "BrcmfSdio.hpp"
+#include "Ieee80211.hpp"
+#include "Wpa.hpp"
+#include <Api/Syscall.hpp>
+#include <Arch/Cpu.hpp>
+#include <CppLib/Stream.hpp>
+#include <Drivers/Platform/RpiFirmware.hpp>
+#include <Libraries/Crypto.hpp>
+#include <Libraries/Memory.hpp>
+#include <Memory/Heap.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// Joining follows brcmf_cfg80211_connect with the supplicant on the host:
+//   "wpaie"    the RSN element our supplicant built (the firmware puts it in
+//              the association request, and it must match message 2's copy)
+//   "wpa_auth" WPA2-PSK (0x80) or PSK-SHA256 (0x8000), 0 for open
+//   "auth"     open system
+//   "wsec"     AES (4) for CCMP, 0 for open
+//   "join"     SSID, BSSID and the chanspec escan reported
+// The firmware answers with E_SET_SSID (status 0 = associated) and E_LINK;
+// the access point then starts the 4-way handshake, whose EAPOL frames come
+// and go as ordinary Ethernet frames. The keys it produces go to the
+// firmware with "wsec_key". The radio does AES-CCMP only: GCMP networks are
+// refused.
+
+using namespace Kt;
+using namespace montauk::abi;
+
+namespace Drivers::Net::Wifi {
+
+    // WLC ioctls
+    static constexpr uint32_t WLC_UP                 = 2;
+    static constexpr uint32_t WLC_SET_INFRA          = 20;
+    static constexpr uint32_t WLC_SET_SSID           = 26;
+    static constexpr uint32_t WLC_SCAN               = 50;
+    static constexpr uint32_t WLC_DISASSOC           = 52;
+    static constexpr uint32_t WLC_SET_PM             = 86;
+    static constexpr uint32_t WLC_GET_VALID_CHANNELS = 217;
+
+    // Events
+    static constexpr uint32_t E_SET_SSID     = 0;
+    static constexpr uint32_t E_AUTH         = 3;
+    static constexpr uint32_t E_DEAUTH       = 5;
+    static constexpr uint32_t E_DEAUTH_IND   = 6;
+    static constexpr uint32_t E_ASSOC        = 7;
+    static constexpr uint32_t E_REASSOC      = 9;
+    static constexpr uint32_t E_DISASSOC     = 11;
+    static constexpr uint32_t E_DISASSOC_IND = 12;
+    static constexpr uint32_t E_LINK         = 16;
+    static constexpr uint32_t E_ROAM         = 19;
+    static constexpr uint32_t E_PSK_SUP      = 46;
+    static constexpr uint32_t E_ESCAN_RESULT = 69;
+
+    static constexpr uint32_t STATUS_SUCCESS = 0;
+    static constexpr uint32_t STATUS_PARTIAL = 8;
+    static constexpr uint16_t EVENT_FLAG_LINK = 0x01;
+
+    static constexpr uint32_t WPA_AUTH_DISABLED   = 0;
+    static constexpr uint32_t WPA2_AUTH_PSK       = 0x80;
+    static constexpr uint32_t WPA2_AUTH_PSK_SHA256= 0x8000;
+    static constexpr uint32_t AES_ENABLED         = 4;
+    static constexpr uint32_t CRYPTO_ALGO_AES_CCM = 4;
+    static constexpr uint32_t KEY_FLAG_PRIMARY    = 1u << 1;
+
+
+    static constexpr uint64_t JOIN_TIMEOUT_MS = 12000;
+    static constexpr uint64_t SCAN_TIMEOUT_MS = 25000;
+    static constexpr uint64_t POLL_ACTIVE_MS  = 10;
+    static constexpr uint64_t POLL_IDLE_MS    = 250;
+
+    // The NVRAM files ship one address for every board; the radio's own
+    // OTP normally overrides it.
+    static constexpr uint8_t NVRAM_PLACEHOLDER_MAC[6] = { 0xb8, 0x27, 0xeb, 0x74, 0xf2, 0x6c };
+
+    // =========================================================================
+    // State
+    // =========================================================================
+
+    static volatile uint32_t g_conn = WIFI_CONN_IDLE;
+    static uint8_t g_bssid[6];
+    static char    g_ssid[33];
+    static uint8_t g_ssidLen = 0;
+    static uint8_t g_channel = 0;
+    static bool    g_secure = false;
+    static bool    g_refusedSecurity = false;
+    static bool    g_authFailed = false;
+    static bool    g_sawEapol = false;
+    static uint64_t g_joinStartMs = 0;
+    static uint64_t g_lastServiceMs = 0;
+    static WpaConfig g_wpaCfg = {};
+
+    // Raised by events inside the pump, consumed by ConnectService().
+    static volatile bool g_evAssoc = false;
+    static volatile bool g_evJoinFailed = false;
+    static volatile bool g_evLinkDown = false;
+    static volatile uint32_t g_evStatus = 0;
+    static volatile uint32_t g_evReason = 0;
+
+    static volatile bool g_scanActive = false;
+    static uint64_t g_scanStartMs = 0;
+    static uint16_t g_syncId = 0;
+    static uint32_t g_scanSeen = 0;
+
+    static volatile bool g_dumpConsole = false;
+    static bool g_failureLogged = false;
+
+    // The chanspec escan reported for each BSS, in the firmware's own
+    // encoding, so the join names the channel exactly.
+    struct BssChan { uint8_t Bssid[6]; uint16_t Chanspec; bool Used; };
+    static BssChan g_chans[64];
+    static kcp::Spinlock g_chanLock;
+
+    struct PendingEapol { uint32_t Len; uint8_t Data[512]; };
+    static constexpr uint32_t EAPOL_DEPTH = 4;
+    static PendingEapol g_eapolQ[EAPOL_DEPTH];
+    static uint32_t g_eapolHead = 0, g_eapolTail = 0;
+    static kcp::Spinlock g_eapolLock;
+
+    static inline uint16_t Le16(const uint8_t* p) { return (uint16_t)(p[0] | p[1] << 8); }
+    static inline uint32_t Le32(const uint8_t* p) {
+        return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24;
+    }
+    static inline void Put32(uint8_t* p, uint32_t v) { memcpy(p, &v, 4); }
+    static inline void Put16(uint8_t* p, uint16_t v) { memcpy(p, &v, 2); }
+
+    static bool Elapsed(uint64_t now, uint64_t since, uint64_t ms) {
+        return now >= since && now - since >= ms;
+    }
+
+    static void PutMac(KernelLogStream& s, const uint8_t* m) {
+        static const char hex[] = "0123456789abcdef";
+        char buf[18];
+        for (int i = 0; i < 6; i++) {
+            buf[i * 3] = hex[m[i] >> 4];
+            buf[i * 3 + 1] = hex[m[i] & 0xF];
+            buf[i * 3 + 2] = i < 5 ? ':' : '\0';
+        }
+        s << buf;
+    }
+
+    // =========================================================================
+    // Events (inside the pump: record, never send)
+    // =========================================================================
+
+    static void RememberChanspec(const uint8_t* bssid, uint16_t chanspec) {
+        g_chanLock.Acquire();
+        BssChan* slot = nullptr;
+        for (auto& c : g_chans) {
+            if (c.Used && memcmp(c.Bssid, bssid, 6) == 0) { slot = &c; break; }
+            if (!c.Used && !slot) slot = &c;
+        }
+        if (slot) {
+            memcpy(slot->Bssid, bssid, 6);
+            slot->Chanspec = chanspec;
+            slot->Used = true;
+        }
+        g_chanLock.Release();
+    }
+
+    static uint16_t LookupChanspec(const uint8_t* bssid, uint8_t channel) {
+        uint16_t found = 0;
+        g_chanLock.Acquire();
+        for (auto& c : g_chans) {
+            if (c.Used && memcmp(c.Bssid, bssid, 6) == 0) { found = c.Chanspec; break; }
+        }
+        g_chanLock.Release();
+        if (found) return found;
+        // 802.11ac encoding: channel, 20 MHz, band.
+        return (uint16_t)(channel | 0x1000 | (channel > 14 ? 0xC000 : 0x0000));
+    }
+
+    // brcmf_escan_result_le: buflen, version, sync_id, bss_count, then
+    // brcmf_bss_info_le records (128 bytes fixed, IEs at ie_offset).
+    static void HandleEscanResult(const uint8_t* data, uint32_t len) {
+        if (len < 12) return;
+        uint16_t count = Le16(data + 10);
+        uint32_t off = 12;
+        for (uint16_t i = 0; i < count; i++) {
+            if (off + 128 > len) break;
+            const uint8_t* bi = data + off;
+            uint32_t biLen = Le32(bi + 4);
+            if (biLen < 128 || biLen > len - off) break;
+
+            const uint8_t* bssid = bi + 8;
+            uint16_t beacon = Le16(bi + 14);
+            uint16_t caps = Le16(bi + 16);
+            uint16_t chanspec = Le16(bi + 72);
+            int16_t rssi = (int16_t)Le16(bi + 78);
+            uint8_t ctlCh = bi[88];
+            uint16_t ieOff = Le16(bi + 116);
+            uint32_t ieLen = Le32(bi + 120);
+            if (ieOff > biLen || ieLen > biLen - ieOff) ieLen = 0;
+
+            uint8_t channel = ctlCh ? ctlCh : (uint8_t)(chanspec & 0xFF);
+            if (rssi < -127) rssi = -127;
+            if (rssi > 0) rssi = 0;
+            RememberChanspec(bssid, chanspec);
+            WifiRxBssInfo(bssid, beacon, caps, ieLen ? bi + ieOff : nullptr, ieLen,
+                          channel, (int8_t)rssi);
+            g_scanSeen++;
+            off += biLen;
+        }
+    }
+
+    void BrcmfOnEvent(uint32_t type, uint32_t status, uint32_t reason, uint16_t flags,
+                      const uint8_t* addr, const uint8_t* data, uint32_t len) {
+        (void)addr;
+        switch (type) {
+            case E_ESCAN_RESULT:
+                if (status == STATUS_PARTIAL) {
+                    HandleEscanResult(data, len);
+                } else if (g_scanActive) {
+                    g_scanActive = false;
+                    KernelLogStream(INFO, "WiFi") << "Scan " << (status == STATUS_SUCCESS ? "complete" : "ended")
+                        << ": " << (uint64_t)g_scanSeen << " results";
+                }
+                break;
+
+            case E_SET_SSID:
+                if (g_conn != WIFI_CONN_ASSOCIATING) break;
+                if (status == STATUS_SUCCESS) {
+                    g_evAssoc = true;
+                } else {
+                    g_evStatus = status;
+                    g_evJoinFailed = true;
+                }
+                break;
+
+            case E_LINK:
+                if (flags & EVENT_FLAG_LINK) {
+                    if (g_conn == WIFI_CONN_ASSOCIATING) g_evAssoc = true;
+                } else if (g_conn >= WIFI_CONN_ASSOCIATED && g_conn != WIFI_CONN_FAILED) {
+                    g_evReason = reason;
+                    g_evLinkDown = true;
+                }
+                break;
+
+            case E_DEAUTH:
+            case E_DEAUTH_IND:
+            case E_DISASSOC_IND:
+                if (g_conn >= WIFI_CONN_ASSOCIATED && g_conn != WIFI_CONN_FAILED) {
+                    g_evReason = reason;
+                    g_evLinkDown = true;
+                }
+                break;
+
+            case E_AUTH:
+                if (status != STATUS_SUCCESS && g_conn == WIFI_CONN_ASSOCIATING)
+                    KernelLogStream(WARNING, "WiFi") << "Authentication rejected (status "
+                        << (uint64_t)status << ", reason " << (uint64_t)reason << ")";
+                break;
+
+            case E_ASSOC:
+            case E_REASSOC:
+                if (status != STATUS_SUCCESS && g_conn == WIFI_CONN_ASSOCIATING)
+                    KernelLogStream(WARNING, "WiFi") << "Association rejected (status "
+                        << (uint64_t)status << ", reason " << (uint64_t)reason << ")";
+                break;
+
+            default:
+                break;
+        }
+    }
+
+    static void QueueEapol(const uint8_t* body, uint32_t len) {
+        if (len > sizeof(g_eapolQ[0].Data)) return;
+        g_eapolLock.Acquire();
+        uint32_t next = (g_eapolHead + 1) % EAPOL_DEPTH;
+        if (next == g_eapolTail) g_eapolTail = (g_eapolTail + 1) % EAPOL_DEPTH;
+        memcpy(g_eapolQ[g_eapolHead].Data, body, len);
+        g_eapolQ[g_eapolHead].Len = len;
+        g_eapolHead = next;
+        g_eapolLock.Release();
+    }
+
+    static bool EapolPending() {
+        g_eapolLock.Acquire();
+        bool pending = g_eapolHead != g_eapolTail;
+        g_eapolLock.Release();
+        return pending;
+    }
+
+    void BrcmfOnRxEthernet(const uint8_t* frame, uint32_t len) {
+        uint16_t etherType = (uint16_t)(frame[12] << 8 | frame[13]);
+        if (etherType == ETHERTYPE_EAPOL) {
+            if (g_secure && g_conn >= WIFI_CONN_ASSOCIATING && g_conn != WIFI_CONN_FAILED) {
+                QueueEapol(frame + 14, len - 14);
+            }
+            return;
+        }
+        if (BrcmfLinkUp()) WifiRxEthernet(frame, len);
+    }
+
+    void BrcmfOnFirmwareFailure(const char* why) {
+        if (g_brcmf.State != BrcmfState::Running) return;
+        g_brcmf.State = BrcmfState::Error;
+        g_brcmf.FwErrors++;
+        g_scanActive = false;
+        g_dumpConsole = true;
+        if (!g_failureLogged) {
+            g_failureLogged = true;
+            KernelLogStream(ERROR, "WiFi") << "Radio failed: " << why;
+        }
+    }
+
+    // =========================================================================
+    // Bring-up
+    // =========================================================================
+
+    // CLM blob: the regulatory database the firmware ships without
+    // ("clm_min"). Sent in 1400-byte chunks behind brcmf_dload_data_le:
+    // flag (handler version 1 << 12 | begin 2 | end 4), type (2 = CLM),
+    // length, CRC (unused), data.
+    static bool LoadClm() {
+        uint32_t size = 0;
+        uint8_t* blob = BrcmfLoadFile("clm_blob", size, false);
+        if (!blob) {
+            KernelLogStream(WARNING, "WiFi") << "No CLM blob; the firmware's own regulatory data is used";
+            return true;
+        }
+
+        constexpr uint32_t CHUNK = 1400;
+        constexpr uint32_t HDR = 12;
+        uint8_t* chunk = (uint8_t*)Memory::g_heap->Request(HDR + CHUNK + 4);
+        bool ok = chunk != nullptr;
+        uint32_t off = 0;
+        uint16_t flag = 1u << 12 | 0x0002;
+        while (ok && off < size) {
+            uint32_t n = size - off > CHUNK ? CHUNK : size - off;
+            if (off + n == size) flag |= 0x0004;
+            memset(chunk, 0, HDR + CHUNK + 4);
+            Put16(chunk, flag);
+            Put16(chunk + 2, 2);
+            Put32(chunk + 4, n);
+            Put32(chunk + 8, 0);
+            memcpy(chunk + HDR, blob + off, n);
+            // Linux sends sizeof(struct) + len - 1: three bytes of padding.
+            ok = BrcmfIovarSet("clmload", chunk, HDR + n + 3);
+            flag &= (uint16_t)~0x0002;
+            off += n;
+        }
+        if (chunk) Memory::g_heap->Free(chunk);
+        Memory::g_heap->Free(blob);
+
+        uint32_t status = 0;
+        if (ok && BrcmfIovarGetInt("clmload_status", status, true) && status != 0) {
+            KernelLogStream(ERROR, "WiFi") << "The firmware rejected the CLM blob (status "
+                << (uint64_t)status << ")";
+            ok = false;
+        }
+        if (ok) KernelLogStream(INFO, "WiFi") << "CLM blob loaded (" << (uint64_t)size << " bytes)";
+        return ok;
+    }
+
+    static void SetEventMask() {
+        uint8_t mask[16] = {};
+        const uint32_t events[] = {
+            E_SET_SSID, E_AUTH, E_DEAUTH, E_DEAUTH_IND, E_ASSOC, E_REASSOC, E_DISASSOC,
+            E_DISASSOC_IND, E_LINK, E_ROAM, E_PSK_SUP, E_ESCAN_RESULT,
+        };
+        for (uint32_t e : events) mask[e / 8] |= (uint8_t)(1u << (e % 8));
+        BrcmfIovarSet("event_msgs", mask, sizeof(mask));
+    }
+
+    // "Version: 7.45.265 (28bca26 CY) CRC: ..." -> "7.45.265"
+    static void ParseVersion(const char* ver) {
+        const char* p = ver;
+        const char* key = "version ";
+        while (*p) {
+            const char* a = p;
+            const char* b = key;
+            while (*a && *b && (*a | 0x20) == *b) { a++; b++; }
+            if (!*b) { p = a; break; }
+            p++;
+        }
+        if (!*p) p = ver;
+        int n = 0;
+        while (p[n] && p[n] != ' ' && p[n] != '\n' && n < 31) {
+            g_brcmf.FwVersion[n] = p[n];
+            n++;
+        }
+        g_brcmf.FwVersion[n] = '\0';
+    }
+
+    static void ReadChannels() {
+        uint32_t list[65] = {};
+        list[0] = 64;
+        g_brcmf.ChannelCount = 0;
+        if (!BrcmfIoctl(WLC_GET_VALID_CHANNELS, false, list, sizeof(list))) return;
+        uint32_t count = list[0] > 64 ? 64 : list[0];
+        bool b24 = false, b5 = false;
+        for (uint32_t i = 0; i < count; i++) {
+            if (list[1 + i] > 14) b5 = true;
+            else if (list[1 + i] > 0) b24 = true;
+        }
+        g_brcmf.ChannelCount = (uint16_t)count;
+        g_brcmf.Band24 = b24;
+        g_brcmf.Band5 = b5;
+    }
+
+    static bool Preinit() {
+        uint8_t mac[6] = {};
+        if (!BrcmfIovarGet("cur_etheraddr", mac, 6)) return false;
+        if (memcmp(mac, NVRAM_PLACEHOLDER_MAC, 6) == 0) {
+            // Raspberry Pi convention: wlan0 is the board (Ethernet) address
+            // plus one.
+            uint8_t board[6];
+            if (Drivers::Platform::RpiFirmware::GetBoardMacAddress(board)) {
+                memcpy(mac, board, 6);
+                for (int i = 5; i >= 3; i--) if (++mac[i] != 0) break;
+                BrcmfIovarSet("cur_etheraddr", mac, 6);
+            }
+        }
+        memcpy(g_brcmf.Mac, mac, 6);
+
+        char ver[256] = {};
+        if (BrcmfIovarGet("ver", ver, sizeof(ver) - 1)) {
+            ver[sizeof(ver) - 1] = '\0';
+            for (char& c : ver) if (c == '\n' || c == '\r') c = ' ';
+            KernelLogStream(INFO, "WiFi") << "Firmware: " << ver;
+            ParseVersion(ver);
+        }
+
+        if (!LoadClm()) return false;
+
+        // No glomming in either direction: one SDPCM frame per transfer.
+        BrcmfIovarSetInt("bus:txglom", 0, true);
+        BrcmfIovarSetInt("bus:rxglom", 0, true);
+
+        BrcmfSetInt(WLC_SET_INFRA, 1);
+        // Roaming would re-key without the host's supplicant; ARP/ND offload
+        // would answer for an address the firmware was never told.
+        BrcmfIovarSetInt("roam_off", 1, true);
+        BrcmfIovarSetInt("arpoe", 0, true);
+        BrcmfIovarSetInt("ndoe", 0, true);
+        // Power save off: the radio misses broadcasts (ARP, DHCP offers)
+        // while dozing, and the Pi is not on a battery.
+        BrcmfSetInt(WLC_SET_PM, 0);
+
+        SetEventMask();
+        if (!BrcmfSetInt(WLC_UP, 1)) {
+            KernelLogStream(ERROR, "WiFi") << "The firmware would not bring the interface up";
+            return false;
+        }
+        ReadChannels();
+        return true;
+    }
+
+    static void ResetConnection() {
+        g_conn = WIFI_CONN_IDLE;
+        g_evAssoc = false;
+        g_evJoinFailed = false;
+        g_evLinkDown = false;
+        g_secure = false;
+        g_sawEapol = false;
+        g_eapolLock.Acquire();
+        g_eapolHead = g_eapolTail = 0;
+        g_eapolLock.Release();
+        WpaReset();
+    }
+
+    bool BrcmfProbe(uint64_t arasanMmio) {
+        if (g_brcmf.State != BrcmfState::Absent) return false;
+        g_brcmf.ArasanMmio = arasanMmio;
+        g_brcmf.State = BrcmfState::Detected;
+        return true;
+    }
+
+    bool BrcmfCompleteInit() {
+        g_brcmf.State = BrcmfState::Booting;
+        g_failureLogged = false;
+        g_dumpConsole = false;
+        g_scanActive = false;
+        g_brcmf.CmdFailBucket = 0;
+        ResetConnection();
+        BrcmfProtoReset();
+
+        // One retry from power-off: the Bluetooth half of the same chip may
+        // be powering up or taking its patch download at the same moment.
+        bool up = BrcmfChipBringUp();
+        if (!up) {
+            KernelLogStream(WARNING, "WiFi") << "Bring-up failed; power-cycling the radio and trying once more";
+            Timekeeping::BusyWaitMs(100);
+            up = BrcmfChipBringUp();
+        }
+        if (!up) {
+            g_brcmf.State = BrcmfState::Error;
+            return false;
+        }
+
+        // Running from here: the preinit below is the first SDPCM traffic.
+        g_brcmf.LastPollMs = Timekeeping::GetMilliseconds();
+        g_brcmf.State = BrcmfState::Running;
+        BrcmfDumpConsole();
+
+        if (!Preinit()) {
+            KernelLogStream(ERROR, "WiFi") << "Firmware initialization failed";
+            BrcmfDumpConsole();
+            g_brcmf.State = BrcmfState::Error;
+            BrcmfChipPowerOff();
+            return false;
+        }
+        BrcmfDumpConsole();
+
+        auto log = KernelLogStream(OK, "WiFi");
+        log << "CYW" << g_brcmf.FwName << " ready: firmware " << g_brcmf.FwVersion << ", "
+            << (uint64_t)g_brcmf.ChannelCount << " channels, MAC ";
+        PutMac(log, g_brcmf.Mac);
+        return true;
+    }
+
+    void BrcmfShutdown() {
+        g_scanActive = false;
+        ResetConnection();
+        BrcmfChipPowerOff();
+        g_brcmf.State = BrcmfState::Detected;
+    }
+
+    BrcmfState BrcmfGetState() { return g_brcmf.State; }
+
+    // =========================================================================
+    // Scanning
+    // =========================================================================
+
+    bool BrcmfScanActive() { return g_scanActive; }
+
+    bool BrcmfStartScan() {
+        if (g_brcmf.State != BrcmfState::Running) return false;
+        if (g_scanActive) return true;
+
+        // brcmf_escan_params_le: version 1, action 1 (start), sync id, then
+        // brcmf_scan_params_le: SSID (any), BSSID (broadcast), BSS type any,
+        // active scan, firmware defaults for every dwell, all channels.
+        uint8_t p[72] = {};
+        Put32(p, 1);
+        Put16(p + 4, 1);
+        Put16(p + 6, ++g_syncId);
+        memset(p + 44, 0xFF, 6);
+        p[50] = 2;
+        p[51] = 0;
+        Put32(p + 52, 0xFFFFFFFF);
+        Put32(p + 56, 0xFFFFFFFF);
+        Put32(p + 60, 0xFFFFFFFF);
+        Put32(p + 64, 0xFFFFFFFF);
+        Put32(p + 68, 0);
+
+        g_scanSeen = 0;
+        g_scanStartMs = Timekeeping::GetMilliseconds();
+        g_scanActive = true;     // results may arrive while the ioctl is pumping
+        if (!BrcmfIovarSet("escan", p, sizeof(p))) {
+            g_scanActive = false;
+            return false;
+        }
+        return true;
+    }
+
+    void BrcmfAbortScan() {
+        if (!g_scanActive) return;
+        // A WLC_SCAN naming channel -1 cancels the scan in progress.
+        uint8_t p[68] = {};
+        memset(p + 36, 0xFF, 6);
+        p[42] = 2;
+        Put32(p + 44, 0xFFFFFFFF);
+        Put32(p + 48, 0xFFFFFFFF);
+        Put32(p + 52, 0xFFFFFFFF);
+        Put32(p + 56, 0xFFFFFFFF);
+        Put32(p + 60, 1);
+        Put16(p + 64, 0xFFFF);
+        BrcmfIoctl(WLC_SCAN, true, p, sizeof(p));
+        g_scanActive = false;
+    }
+
+    // =========================================================================
+    // Joining
+    // =========================================================================
+
+    int BrcmfConnectState() { return (int)g_conn; }
+    bool BrcmfConnectRefusedForSecurity() { return g_refusedSecurity; }
+    bool BrcmfJoinFailedOnAuth() { return g_authFailed || WpaGetState() == WpaState::Failed; }
+    const char* BrcmfConnectSsid() { return g_ssid; }
+    const uint8_t* BrcmfConnectBssid() { return g_bssid; }
+    uint8_t BrcmfConnectChannel() { return g_channel; }
+    bool BrcmfLinkUp() { return g_brcmf.State == BrcmfState::Running && g_conn == WIFI_CONN_CONNECTED; }
+
+    static void Disassociate() {
+        uint8_t scb[12] = {};
+        Put32(scb, 3);                   // deauthenticated: leaving
+        memcpy(scb + 4, g_bssid, 6);
+        BrcmfIoctl(WLC_DISASSOC, true, scb, sizeof(scb));
+    }
+
+    void BrcmfConnectAbort() {
+        if (g_conn != WIFI_CONN_IDLE && g_brcmf.State == BrcmfState::Running) Disassociate();
+        ResetConnection();
+    }
+
+    bool BrcmfConnectStart(const uint8_t* bssid, uint8_t channel, const char* ssid,
+                           const char* password, const uint8_t* rsnIe, uint8_t rsnIeLen) {
+        if (g_brcmf.State != BrcmfState::Running) return false;
+        if (g_conn != WIFI_CONN_IDLE) BrcmfConnectAbort();
+        g_refusedSecurity = false;
+        g_authFailed = false;
+
+        memcpy(g_bssid, bssid, 6);
+        g_channel = channel;
+        g_ssidLen = 0;
+        while (ssid && g_ssidLen < 32 && ssid[g_ssidLen]) {
+            g_ssid[g_ssidLen] = ssid[g_ssidLen];
+            g_ssidLen++;
+        }
+        g_ssid[g_ssidLen] = '\0';
+
+        bool secure = rsnIe && rsnIeLen > 0;
+        uint8_t ie[64];
+        uint32_t ieLen = 0;
+        if (secure) {
+            memset(&g_wpaCfg, 0, sizeof(g_wpaCfg));
+            if (!WpaParseApRsn(rsnIe, rsnIeLen, g_wpaCfg)) {
+                g_refusedSecurity = true;
+                return false;
+            }
+            if (g_wpaCfg.PairwiseCipher != RSN_CIPHER_CCMP || g_wpaCfg.GroupCipher != RSN_CIPHER_CCMP) {
+                KernelLogStream(WARNING, "WiFi") << "This radio does AES-CCMP only; the network uses GCMP";
+                g_refusedSecurity = true;
+                return false;
+            }
+            uint32_t pl = 0;
+            if (password) while (pl < sizeof(g_wpaCfg.Passphrase) - 1 && password[pl]) pl++;
+            if (pl == 0) {
+                KernelLogStream(WARNING, "WiFi") << "This network is encrypted and needs a passphrase";
+                g_refusedSecurity = true;
+                return false;
+            }
+            memcpy(g_wpaCfg.Passphrase, password, pl);
+            g_wpaCfg.Passphrase[pl] = '\0';
+            g_wpaCfg.PassLen = (uint8_t)pl;
+            memcpy(g_wpaCfg.OwnMac, g_brcmf.Mac, 6);
+            memcpy(g_wpaCfg.Bssid, g_bssid, 6);
+            memcpy(g_wpaCfg.Ssid, g_ssid, g_ssidLen);
+            g_wpaCfg.SsidLen = g_ssidLen;
+            if (!WpaStart(g_wpaCfg)) {
+                g_refusedSecurity = true;
+                return false;
+            }
+            ieLen = WpaBuildRsnIe(ie, sizeof(ie));
+            if (ieLen == 0) return false;
+        }
+
+        if (g_scanActive) BrcmfAbortScan();
+
+        bool sha256 = secure && g_wpaCfg.Akm == RSN_AKM_PSK_SHA256;
+        if (secure) BrcmfIovarSet("wpaie", ie, ieLen);
+        else BrcmfIovarSet("wpaie", nullptr, 0, true);
+        bool ok = BrcmfIovarSetInt("wpa_auth", secure ? (sha256 ? WPA2_AUTH_PSK_SHA256 : WPA2_AUTH_PSK)
+                                                      : WPA_AUTH_DISABLED)
+               && BrcmfIovarSetInt("auth", 0)
+               && BrcmfIovarSetInt("wsec", secure ? AES_ENABLED : 0);
+        BrcmfIovarSetInt("mfp", 0, true);
+        BrcmfIovarSetInt("sup_wpa", 0, true);
+        if (!ok) {
+            WpaReset();
+            return false;
+        }
+
+        uint16_t chanspec = LookupChanspec(g_bssid, channel);
+
+        g_secure = secure;
+        g_evAssoc = false;
+        g_evJoinFailed = false;
+        g_evLinkDown = false;
+        g_joinStartMs = Timekeeping::GetMilliseconds();
+        g_conn = WIFI_CONN_ASSOCIATING;   // before "join": its events may land while it pumps
+
+        // brcmf_ext_join_params_le: SSID; join scan (type default, probes
+        // and dwells as Linux uses for a known channel); BSSID, one chanspec.
+        uint8_t jp[72] = {};
+        Put32(jp, g_ssidLen);
+        memcpy(jp + 4, g_ssid, g_ssidLen);
+        jp[36] = 0xFF;
+        Put32(jp + 40, 16);
+        Put32(jp + 44, 320);
+        Put32(jp + 48, 400);
+        Put32(jp + 52, 0xFFFFFFFF);
+        memcpy(jp + 56, g_bssid, 6);
+        Put32(jp + 64, 1);
+        Put16(jp + 68, chanspec);
+
+        if (!BrcmfIovarSet("join", jp, sizeof(jp), true)) {
+            // Older firmware: WLC_SET_SSID with brcmf_join_params_le.
+            uint8_t sp[52] = {};
+            Put32(sp, g_ssidLen);
+            memcpy(sp + 4, g_ssid, g_ssidLen);
+            memcpy(sp + 36, g_bssid, 6);
+            Put32(sp + 44, 1);
+            Put16(sp + 48, chanspec);
+            if (!BrcmfIoctl(WLC_SET_SSID, true, sp, sizeof(sp))) {
+                KernelLogStream(ERROR, "WiFi") << "The firmware refused the join request";
+                ResetConnection();
+                return false;
+            }
+        }
+
+        auto log = KernelLogStream(INFO, "WiFi");
+        log << "Joining \"" << g_ssid << "\" (";
+        PutMac(log, g_bssid);
+        log << ", channel " << (uint64_t)channel << ", chanspec 0x" << base::hex << (uint64_t)chanspec
+            << base::dec << (secure ? ", WPA2-PSK)" : ", open)");
+        return true;
+    }
+
+    // =========================================================================
+    // Keys and EAPOL (the supplicant's hooks)
+    // =========================================================================
+
+    bool BrcmfWpaTxEapol(const uint8_t* body, uint32_t len) {
+        uint8_t frame[14 + 512];
+        if (len > 512) return false;
+        memcpy(frame, g_bssid, 6);
+        memcpy(frame + 6, g_brcmf.Mac, 6);
+        frame[12] = ETHERTYPE_EAPOL >> 8;
+        frame[13] = ETHERTYPE_EAPOL & 0xFF;
+        memcpy(frame + 14, body, len);
+        return BrcmfTxEthernet(frame, 14 + len);
+    }
+
+    // brcmf_wsec_key_le: index, len, data[32], pad[18], algo, flags, pad[3],
+    // iv_initialized, pad, rxiv {hi, lo}, pad[2], ea[6] -- 164 bytes.
+    static bool SendKey(uint32_t index, const uint8_t* key, uint32_t keyLen, uint32_t flags,
+                        const uint8_t* ea, const uint8_t* rsc) {
+        if (keyLen > 32) return false;
+        uint8_t k[164] = {};
+        Put32(k, index);
+        Put32(k + 4, keyLen);
+        memcpy(k + 8, key, keyLen);
+        Put32(k + 112, CRYPTO_ALGO_AES_CCM);
+        Put32(k + 116, flags);
+        if (rsc) {
+            Put32(k + 132, 1);
+            Put32(k + 140, (uint32_t)rsc[2] | (uint32_t)rsc[3] << 8 | (uint32_t)rsc[4] << 16
+                         | (uint32_t)rsc[5] << 24);
+            Put16(k + 144, (uint16_t)(rsc[0] | rsc[1] << 8));
+        }
+        if (ea) memcpy(k + 156, ea, 6);
+        bool ok = BrcmfIovarSet("wsec_key", k, sizeof(k));
+        Crypto::SecureZero(k, sizeof(k));
+        return ok;
+    }
+
+    // Message 4 is still in the transmit queue when the supplicant installs
+    // the pairwise key; it has to reach the radio first or it goes out
+    // encrypted and the access point never sees it (Linux waits for its
+    // pending 802.1X frames the same way).
+    static void DrainTx() {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (BrcmfTxPending() && Timekeeping::GetMilliseconds() - start < 200) {
+            BrcmfPoll();
+            Timekeeping::BusyWaitUs(200);
+        }
+        Timekeeping::BusyWaitMs(10);
+    }
+
+    bool BrcmfWpaInstallPtk(const uint8_t* tk, uint32_t tkLen, uint8_t cipher) {
+        if (cipher != RSN_CIPHER_CCMP) return false;
+        DrainTx();
+        return SendKey(0, tk, tkLen, 0, g_bssid, nullptr);
+    }
+
+    bool BrcmfWpaInstallGtk(const uint8_t* gtk, uint32_t gtkLen, uint8_t keyIdx,
+                            uint8_t cipher, const uint8_t* rsc) {
+        if (cipher != RSN_CIPHER_CCMP) return false;
+        DrainTx();
+        return SendKey(keyIdx, gtk, gtkLen, KEY_FLAG_PRIMARY, nullptr, rsc);
+    }
+
+    // =========================================================================
+    // Service (idle loop / syscall waits: may send ioctls)
+    // =========================================================================
+
+    static void ConnectService() {
+        static volatile bool inService = false;
+        if (__atomic_test_and_set(&inService, __ATOMIC_ACQUIRE)) return;
+        struct Guard { ~Guard() { __atomic_clear(&inService, __ATOMIC_RELEASE); } } guard;
+
+        uint64_t now = Timekeeping::GetMilliseconds();
+        g_lastServiceMs = now;
+
+        if (g_scanActive && Elapsed(now, g_scanStartMs, SCAN_TIMEOUT_MS)) {
+            KernelLogStream(WARNING, "WiFi") << "Scan never finished; abandoning it";
+            BrcmfAbortScan();
+        }
+
+        if (g_conn == WIFI_CONN_ASSOCIATING) {
+            if (g_evJoinFailed) {
+                g_evJoinFailed = false;
+                KernelLogStream(WARNING, "WiFi") << "Could not join \"" << g_ssid << "\" (status "
+                    << (uint64_t)g_evStatus << (g_evStatus == 3 ? ": network not found)" : ")");
+                g_conn = WIFI_CONN_FAILED;
+            } else if (g_evAssoc) {
+                g_evAssoc = false;
+                if (g_secure) {
+                    KernelLogStream(OK, "WiFi") << "Associated with \"" << g_ssid << "\"; starting the key handshake";
+                    g_conn = WIFI_CONN_HANDSHAKING;
+                } else {
+                    KernelLogStream(OK, "WiFi") << "Connected to \"" << g_ssid << "\" (open network)";
+                    g_conn = WIFI_CONN_CONNECTED;
+                }
+            } else if (Elapsed(Timekeeping::GetMilliseconds(), g_joinStartMs, JOIN_TIMEOUT_MS)) {
+                KernelLogStream(WARNING, "WiFi") << "Timed out joining \"" << g_ssid << "\"";
+                g_conn = WIFI_CONN_FAILED;
+            }
+        }
+
+        if (g_evLinkDown && g_conn >= WIFI_CONN_ASSOCIATED && g_conn != WIFI_CONN_FAILED) {
+            g_evLinkDown = false;
+            if (g_conn == WIFI_CONN_HANDSHAKING) {
+                // The access point gives up on a handshake whose MICs do not
+                // verify: a wrong passphrase, nearly always.
+                KernelLogStream(WARNING, "WiFi") << "The access point dropped the connection during the key "
+                    "handshake (reason " << (uint64_t)g_evReason << "); check the passphrase";
+                g_authFailed = g_sawEapol;
+                g_conn = WIFI_CONN_FAILED;
+            } else {
+                KernelLogStream(WARNING, "WiFi") << "Disconnected from \"" << g_ssid << "\" (reason "
+                    << (uint64_t)g_evReason << ")";
+                ResetConnection();
+            }
+        }
+
+        if (g_conn == WIFI_CONN_HANDSHAKING || g_conn == WIFI_CONN_CONNECTED) {
+            uint8_t body[512];
+            for (;;) {
+                uint32_t len = 0;
+                g_eapolLock.Acquire();
+                if (g_eapolHead != g_eapolTail) {
+                    len = g_eapolQ[g_eapolTail].Len;
+                    memcpy(body, g_eapolQ[g_eapolTail].Data, len);
+                    g_eapolTail = (g_eapolTail + 1) % EAPOL_DEPTH;
+                }
+                g_eapolLock.Release();
+                if (!len) break;
+                g_sawEapol = true;
+                WpaOnEapol(body, len);
+            }
+        }
+
+        if (g_conn == WIFI_CONN_HANDSHAKING) {
+            WpaService(Timekeeping::GetMilliseconds());
+            if (WpaIsComplete()) {
+                KernelLogStream(OK, "WiFi") << "Connected to \"" << g_ssid << "\"";
+                g_conn = WIFI_CONN_CONNECTED;
+            } else if (WpaGetState() == WpaState::Failed) {
+                g_authFailed = true;
+                g_conn = WIFI_CONN_FAILED;
+            }
+        }
+    }
+
+    bool BrcmfHasWork(uint64_t now) {
+        if (g_brcmf.State == BrcmfState::Error) return g_dumpConsole;
+        if (g_brcmf.State != BrcmfState::Running) return false;
+
+        bool active = g_scanActive || (g_conn != WIFI_CONN_IDLE && g_conn != WIFI_CONN_FAILED);
+        if (BrcmfPollDue(now, active ? POLL_ACTIVE_MS : POLL_IDLE_MS)) return true;
+        if (g_evAssoc || g_evJoinFailed || g_evLinkDown || EapolPending()) return true;
+        // Deadlines and the supplicant's retransmissions, even with the air quiet.
+        if ((g_conn == WIFI_CONN_ASSOCIATING || g_conn == WIFI_CONN_HANDSHAKING || g_scanActive)
+            && Elapsed(now, g_lastServiceMs, 50))
+            return true;
+        return false;
+    }
+
+    void BrcmfService() {
+        if (g_brcmf.State == BrcmfState::Error) {
+            if (g_dumpConsole) {
+                g_dumpConsole = false;
+                BrcmfDumpConsole();
+            }
+            return;
+        }
+        if (g_brcmf.State != BrcmfState::Running) return;
+        BrcmfPoll();
+        ConnectService();
+    }
+
+    bool BrcmfSendEthernet(const uint8_t* frame, uint32_t len) {
+        if (!BrcmfLinkUp()) return false;
+        return BrcmfTxEthernet(frame, len);
+    }
+
+}

kernel/src/Drivers/Net/Wifi/BrcmfProto.cpp

@@ -0,0 +1,592 @@
+/*
+    * BrcmfProto.cpp
+    * Broadcom FullMAC over SDIO: SDPCM framing, CDC ioctls, BDC data frames,
+    * firmware events, the RX pump and the transmit queue
+    * Derived in part from Linux's brcmfmac, Copyright (c) 2010-2014 Broadcom
+    * Corporation, ISC license (notice in programs/data/licenses/NOTICES.txt).
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Brcmf.hpp"
+#include "BrcmfSdio.hpp"
+#include <Arch/Cpu.hpp>
+#include <CppLib/Stream.hpp>
+#include <Libraries/Memory.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+#include <atomic>
+
+// Every frame on function 2 starts with the SDPCM header (Linux sdio.c):
+//
+//   hardware  u16 len (the whole frame), u16 ~len
+//   software  u8 seq, u8 channel (low nibble; bit 7 = glom descriptor),
+//             u8 next length, u8 data offset (from the frame start),
+//             u8 flow control bitmap, u8 max seq (TX credit, inbound),
+//             u16 reserved
+//
+// Channel 0 carries ioctls and their answers behind a 16-byte CDC header
+// (cmd, len, flags: id [31:16] | interface [15:12] | set [1] | error [0],
+// status). Channels 1 (events) and 2 (data) carry Ethernet frames behind a
+// 4-byte BDC header (flags: version 2 in [7:4]; priority; flags2: interface;
+// data offset in dwords to skip). Events are Ethernet frames of type 0x886C
+// with Broadcom's OUI.
+//
+// Transmit credit: the device's max seq bounds what may be sent, every frame
+// (ioctls included) takes the next sequence number.
+
+using namespace Kt;
+
+namespace Drivers::Net::Wifi {
+
+    static constexpr uint32_t SDPCM_HDRLEN   = 12;
+    static constexpr uint8_t  CHAN_CONTROL   = 0;
+    static constexpr uint8_t  CHAN_EVENT     = 1;
+    static constexpr uint8_t  CHAN_DATA      = 2;
+    static constexpr uint8_t  CHAN_GLOM      = 3;
+    static constexpr uint8_t  GLOM_DESC      = 0x80;
+
+    static constexpr uint32_t CDC_HDRLEN     = 16;
+    static constexpr uint32_t CDCF_ERROR     = 0x01;
+    static constexpr uint32_t CDCF_SET       = 0x02;
+    static constexpr uint32_t BDC_HDRLEN     = 4;
+    static constexpr uint8_t  BDC_VERSION    = 2;
+
+    static constexpr uint16_t ETHERTYPE_BRCM = 0x886C;
+
+    // SDIO device core
+    static constexpr uint32_t SD_INTSTATUS         = 0x20;
+    static constexpr uint32_t SD_TOSBMAILBOX       = 0x40;
+    static constexpr uint32_t SD_TOHOSTMAILBOXDATA = 0x4C;
+    static constexpr uint32_t I_HMB_FRAME_IND      = 1u << 6;
+    static constexpr uint32_t I_HMB_HOST_INT       = 1u << 7;
+    static constexpr uint32_t SMB_INT_ACK          = 1u << 1;
+    static constexpr uint32_t HMB_DATA_FWHALT      = 0x10;
+
+    static constexpr uint32_t F1_FRAMECTRL   = 0x1000D;
+    static constexpr uint32_t F1_RFRAMEBCLO  = 0x1001B;
+    static constexpr uint32_t F1_RFRAMEBCHI  = 0x1001C;
+    static constexpr uint8_t  SFC_RF_TERM    = 0x01;
+
+    // Function 2's FIFO; the address is ChipCommon's window | 4-byte access.
+    static constexpr uint32_t F2_ADDR = 0x8000;
+    static constexpr uint32_t SI_ENUM_BASE = 0x18000000;
+
+    static constexpr uint32_t RX_MAX = 8192;
+    static constexpr uint32_t CTRL_MAX = 2048;              // ioctl payload
+    static constexpr uint32_t TX_SLOT = 2048;               // padded data frame
+    static constexpr uint32_t TXQ_DEPTH = 32;
+    static constexpr uint64_t CREDIT_TIMEOUT_MS = 1000;
+    static constexpr uint64_t IOCTL_TIMEOUT_MS = 2000;
+    static constexpr uint32_t FAIL_BUCKET_LIMIT = 3;
+    static constexpr uint32_t BUS_ERROR_LIMIT = 16;
+
+    static uint32_t g_rxBuf[RX_MAX / 4 + 16];
+    static volatile bool g_pumping = false;
+    static uint32_t g_busErrors = 0;
+    static bool g_glomWarned = false;
+
+    struct TxSlot {
+        uint32_t Len;          // bytes on the bus (padded)
+        uint32_t Data[TX_SLOT / 4];
+    };
+    static TxSlot g_txq[TXQ_DEPTH];
+    static uint32_t g_txHead = 0, g_txTail = 0;
+    static kcp::Spinlock g_txqLock;
+
+    // The outstanding ioctl, if any. The pump on whichever core finds the
+    // answer copies it here under g_respLock; the caller copies it out.
+    static kcp::Mutex g_ctrlLock;
+    static kcp::Spinlock g_respLock;
+    // Room for the largest ioctl padded to whole blocks.
+    static uint32_t g_ctrlTx[(SDPCM_HDRLEN + CDC_HDRLEN + CTRL_MAX + Sdio::FN2_BLOCK - 1)
+                             / Sdio::FN2_BLOCK * Sdio::FN2_BLOCK / 4];
+    static uint8_t g_ctrlResp[CTRL_MAX];
+    static uint32_t g_ctrlRespLen = 0;
+    static uint16_t g_reqId = 0;
+    static uint16_t g_waitId = 0;           // 0 = nobody waiting
+    static int32_t g_ctrlStatus = 0;
+    static uint32_t g_ctrlFlags = 0;
+    static std::atomic<bool> g_ctrlDone{false};
+
+    static inline uint16_t Be16(const uint8_t* p) { return (uint16_t)(p[0] << 8 | p[1]); }
+    static inline uint32_t Be32(const uint8_t* p) {
+        return (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | p[3];
+    }
+    static inline void Le32(uint8_t* p, uint32_t v) { memcpy(p, &v, 4); }
+
+    void BrcmfProtoReset() {
+        g_brcmf.TxSeq = 0;
+        g_brcmf.TxMax = 1;
+        g_brcmf.RxPending = false;
+        g_txqLock.Acquire();
+        g_txHead = g_txTail = 0;
+        g_txqLock.Release();
+        g_respLock.Acquire();
+        g_waitId = 0;
+        g_respLock.Release();
+        g_busErrors = 0;
+        g_glomWarned = false;
+    }
+
+    // =========================================================================
+    // Bus transfers (bus lock held)
+    // =========================================================================
+
+    static bool CanSend() {
+        uint8_t room = (uint8_t)(g_brcmf.TxMax - g_brcmf.TxSeq);
+        return room != 0 && (room & 0x80) == 0;
+    }
+
+    static void NoteBusError() {
+        if (++g_busErrors == BUS_ERROR_LIMIT) BrcmfOnFirmwareFailure("SDIO bus errors");
+    }
+
+    static bool SendFrameLocked(uint8_t* frame, uint32_t busLen) {
+        frame[4] = g_brcmf.TxSeq;
+        if (!BrcmfBpSelectWindow(SI_ENUM_BASE) ||
+            !Sdio::WriteBlock(Sdio::FN2, F2_ADDR, false, frame, busLen)) {
+            NoteBusError();
+            return false;
+        }
+        g_busErrors = 0;
+        g_brcmf.TxSeq++;
+        return true;
+    }
+
+    // Bytes on the bus for a frame: dwords, and whole blocks once it is
+    // longer than one (the device takes the length from the header).
+    static uint32_t BusLength(uint32_t len) {
+        if (len > Sdio::FN2_BLOCK)
+            return (len + Sdio::FN2_BLOCK - 1) / Sdio::FN2_BLOCK * Sdio::FN2_BLOCK;
+        return (len + 3) & ~3u;
+    }
+
+    static void PutSdpcm(uint8_t* f, uint32_t len, uint8_t channel) {
+        uint16_t l = (uint16_t)len;
+        f[0] = (uint8_t)l; f[1] = (uint8_t)(l >> 8);
+        f[2] = (uint8_t)~l; f[3] = (uint8_t)(~l >> 8);
+        f[4] = 0;                       // seq, filled in at send time
+        f[5] = channel;
+        f[6] = 0;
+        f[7] = SDPCM_HDRLEN;
+        f[8] = f[9] = f[10] = f[11] = 0;
+    }
+
+    // Abandon a frame half read: abort function 2, have the device drop the
+    // rest of it, and wait for its read FIFO to empty (Linux rxfail).
+    static void RxFailLocked() {
+        Sdio::Abort(Sdio::FN2);
+        BrcmfF1Write8(F1_FRAMECTRL, SFC_RF_TERM);
+        for (int i = 0; i < 100; i++) {
+            uint8_t hi = 0, lo = 0;
+            if (!BrcmfF1Read8(F1_RFRAMEBCHI, hi) || !BrcmfF1Read8(F1_RFRAMEBCLO, lo)) break;
+            if (hi == 0 && lo == 0) break;
+        }
+        NoteBusError();
+    }
+
+    // Read one frame into g_rxBuf. 1 = a frame, 0 = none waiting, -1 = error.
+    static int ReadFrameLocked(uint32_t& len) {
+        uint8_t* buf = (uint8_t*)g_rxBuf;
+        if (!BrcmfBpSelectWindow(SI_ENUM_BASE) ||
+            !Sdio::ReadBlock(Sdio::FN2, F2_ADDR, false, buf, 4)) {
+            RxFailLocked();
+            return -1;
+        }
+        uint16_t l = (uint16_t)(buf[0] | buf[1] << 8);
+        uint16_t check = (uint16_t)(buf[2] | buf[3] << 8);
+        if ((l | check) == 0) return 0;
+        if ((uint16_t)(l ^ check) != 0xFFFF || l < SDPCM_HDRLEN || l > RX_MAX) {
+            KernelLogStream(WARNING, "WiFi") << "Bad SDPCM header (len 0x" << base::hex
+                << (uint64_t)l << ", check 0x" << (uint64_t)check << base::dec << ")";
+            RxFailLocked();
+            return -1;
+        }
+        uint32_t rest = ((l + 3u) & ~3u) - 4;
+        if (rest && !Sdio::ReadBlock(Sdio::FN2, F2_ADDR, false, buf + 4, rest)) {
+            RxFailLocked();
+            return -1;
+        }
+        g_busErrors = 0;
+        len = l;
+        return 1;
+    }
+
+    static void FlushTxLocked() {
+        for (;;) {
+            g_txqLock.Acquire();
+            bool empty = g_txHead == g_txTail;
+            g_txqLock.Release();
+            if (empty || !CanSend()) return;
+
+            TxSlot& slot = g_txq[g_txTail % TXQ_DEPTH];
+            if (SendFrameLocked((uint8_t*)slot.Data, slot.Len)) g_brcmf.TxPackets++;
+
+            g_txqLock.Acquire();
+            g_txTail++;
+            g_txqLock.Release();
+        }
+    }
+
+    // =========================================================================
+    // Receive dispatch (bus lock released)
+    // =========================================================================
+
+    static void HandleControl(const uint8_t* p, uint32_t len) {
+        if (len < CDC_HDRLEN) return;
+        uint32_t cmd, dlen, flags, status;
+        memcpy(&cmd, p, 4);
+        memcpy(&dlen, p + 4, 4);
+        memcpy(&flags, p + 8, 4);
+        memcpy(&status, p + 12, 4);
+        (void)cmd;
+        (void)dlen;
+        uint16_t id = (uint16_t)(flags >> 16);
+
+        g_respLock.Acquire();
+        if (g_waitId != 0 && id == g_waitId) {
+            uint32_t n = len - CDC_HDRLEN;
+            if (n > CTRL_MAX) n = CTRL_MAX;
+            memcpy(g_ctrlResp, p + CDC_HDRLEN, n);
+            g_ctrlRespLen = n;
+            g_ctrlStatus = (int32_t)status;
+            g_ctrlFlags = flags;
+            g_waitId = 0;
+            g_ctrlDone.store(true, std::memory_order_release);
+        }
+        g_respLock.Release();
+    }
+
+    static void HandleEvent(const uint8_t* eth, uint32_t len) {
+        // Ethernet (14), bcm_ethhdr (10): subtype 0x8001, length, version 1,
+        // OUI 00:10:18, user subtype 1; then the 48-byte event message.
+        constexpr uint32_t MSG = 24;
+        constexpr uint32_t DATA = MSG + 48;
+        if (len < DATA) return;
+        const uint8_t* h = eth + 14;
+        if (Be16(h) != 0x8001 || h[5] != 0x00 || h[6] != 0x10 || h[7] != 0x18 || Be16(h + 8) != 1)
+            return;
+
+        const uint8_t* m = eth + MSG;
+        uint16_t flags = Be16(m + 2);
+        uint32_t type = Be32(m + 4);
+        uint32_t status = Be32(m + 8);
+        uint32_t reason = Be32(m + 12);
+        uint32_t dataLen = Be32(m + 20);
+        const uint8_t* addr = m + 24;
+        uint8_t ifidx = m[46];
+        if (ifidx != 0) return;
+        if (dataLen > len - DATA) dataLen = len - DATA;
+        BrcmfOnEvent(type, status, reason, flags, addr, eth + DATA, dataLen);
+    }
+
+    static void Dispatch(const uint8_t* f, uint32_t len) {
+        const uint8_t* sw = f + 4;
+        uint8_t channel = sw[1] & 0x0F;
+        uint8_t offset = sw[3];
+
+        // Credit: the device's max seq, sanity checked as Linux does.
+        uint8_t max = sw[5];
+        if ((uint8_t)(max - g_brcmf.TxSeq) > 0x40) max = (uint8_t)(g_brcmf.TxSeq + 2);
+        g_brcmf.TxMax = max;
+
+        if ((sw[1] & GLOM_DESC) || channel == CHAN_GLOM) {
+            if (!g_glomWarned) {
+                g_glomWarned = true;
+                KernelLogStream(WARNING, "WiFi") << "Dropping a glommed superframe (rx glom is off)";
+            }
+            return;
+        }
+        if (offset < SDPCM_HDRLEN || offset > len) return;
+        const uint8_t* p = f + offset;
+        uint32_t plen = len - offset;
+        if (plen == 0) return;           // credit update only
+
+        if (channel == CHAN_CONTROL) {
+            HandleControl(p, plen);
+            return;
+        }
+        if (channel != CHAN_EVENT && channel != CHAN_DATA) return;
+
+        if (plen < BDC_HDRLEN || (p[0] >> 4) != BDC_VERSION) return;
+        if ((p[2] & 0x0F) != 0) return;  // another interface
+        uint32_t skip = BDC_HDRLEN + (uint32_t)p[3] * 4;
+        if (skip + 14 > plen) return;
+        const uint8_t* eth = p + skip;
+        uint32_t elen = plen - skip;
+
+        if (Be16(eth + 12) == ETHERTYPE_BRCM) {
+            HandleEvent(eth, elen);
+            return;
+        }
+        if (channel == CHAN_DATA) {
+            g_brcmf.RxPackets++;
+            BrcmfOnRxEthernet(eth, elen);
+        }
+    }
+
+    // =========================================================================
+    // The pump
+    // =========================================================================
+
+    bool BrcmfTxPending() {
+        g_txqLock.Acquire();
+        bool pending = g_txHead != g_txTail;
+        g_txqLock.Release();
+        return pending;
+    }
+
+    bool BrcmfPollDue(uint64_t now, uint64_t intervalMs) {
+        if (g_brcmf.State != BrcmfState::Running) return false;
+        if (g_brcmf.RxPending || BrcmfTxPending()) return true;
+        if (Sdio::CardInterruptPending()) return true;
+        uint64_t last = g_brcmf.LastPollMs;
+        return now < last || now - last >= intervalMs;
+    }
+
+    void BrcmfPoll() {
+        if (g_brcmf.State != BrcmfState::Running) return;
+        if (__atomic_test_and_set(&g_pumping, __ATOMIC_ACQUIRE)) return;
+        struct Guard { ~Guard() { __atomic_clear(&g_pumping, __ATOMIC_RELEASE); } } guard;
+
+        const uint32_t sdio = g_brcmf.SdioCoreBase;
+        for (int round = 0; round < 8 && g_brcmf.State == BrcmfState::Running; round++) {
+            bool frames = g_brcmf.RxPending;
+            uint32_t intStatus = 0;
+            bool halted = false;
+
+            g_brcmf.BusLock.Acquire();
+            if (BrcmfBpRead32(sdio + SD_INTSTATUS, intStatus)) {
+                intStatus &= g_brcmf.HostIntMask;
+                if (intStatus) BrcmfBpWrite32(sdio + SD_INTSTATUS, intStatus);
+                if (intStatus & I_HMB_HOST_INT) {
+                    uint32_t mbox = 0;
+                    BrcmfBpRead32(sdio + SD_TOHOSTMAILBOXDATA, mbox);
+                    BrcmfBpWrite32(sdio + SD_TOSBMAILBOX, SMB_INT_ACK);
+                    halted = (mbox & HMB_DATA_FWHALT) != 0;
+                }
+                if (intStatus & I_HMB_FRAME_IND) frames = true;
+            } else {
+                NoteBusError();
+            }
+            g_brcmf.BusLock.Release();
+            g_brcmf.LastPollMs = Timekeeping::GetMilliseconds();
+
+            if (halted) {
+                BrcmfOnFirmwareFailure("firmware halted");
+                return;
+            }
+
+            for (int n = 0; frames && n < 32; n++) {
+                uint32_t len = 0;
+                g_brcmf.BusLock.Acquire();
+                int r = ReadFrameLocked(len);
+                g_brcmf.BusLock.Release();
+                if (r <= 0) {
+                    frames = false;
+                    break;
+                }
+                Dispatch((const uint8_t*)g_rxBuf, len);
+            }
+            g_brcmf.RxPending = frames;
+
+            if (BrcmfTxPending()) {
+                g_brcmf.BusLock.Acquire();
+                FlushTxLocked();
+                g_brcmf.BusLock.Release();
+            }
+
+            if (!frames && intStatus == 0) break;
+        }
+    }
+
+    // =========================================================================
+    // Transmit
+    // =========================================================================
+
+    bool BrcmfTxEthernet(const uint8_t* frame, uint32_t len) {
+        if (g_brcmf.State != BrcmfState::Running || len < 14 || len > 1518) return false;
+        uint32_t total = SDPCM_HDRLEN + BDC_HDRLEN + len;
+        uint32_t busLen = BusLength(total);
+
+        g_txqLock.Acquire();
+        if (g_txHead - g_txTail >= TXQ_DEPTH) {
+            g_txqLock.Release();
+            return false;
+        }
+        TxSlot& slot = g_txq[g_txHead % TXQ_DEPTH];
+        uint8_t* f = (uint8_t*)slot.Data;
+        PutSdpcm(f, total, CHAN_DATA);
+        f[12] = BDC_VERSION << 4;
+        f[13] = 0;                       // priority: best effort
+        f[14] = 0;                       // interface 0
+        f[15] = 0;                       // no signalling TLVs
+        memcpy(f + 16, frame, len);
+        memset(f + total, 0, busLen - total);
+        slot.Len = busLen;
+        g_txHead++;
+        g_txqLock.Release();
+
+        // Never wait for the bus here: the caller may be the network stack
+        // inside a frame the pump is delivering, or a process that just
+        // preempted the bus lock's holder. Whoever holds it, or the next
+        // pump, sends what is queued.
+        if (g_brcmf.BusLock.TryAcquire()) {
+            FlushTxLocked();
+            g_brcmf.BusLock.Release();
+        }
+        return true;
+    }
+
+    // =========================================================================
+    // Control
+    // =========================================================================
+
+    bool BrcmfIoctl(uint32_t cmd, bool set, void* buf, uint32_t len, int32_t* fwStatus) {
+        if (fwStatus) *fwStatus = 0;
+        if (g_brcmf.State != BrcmfState::Running || len > CTRL_MAX) return false;
+
+        g_ctrlLock.Acquire();
+
+        uint16_t id = ++g_reqId;
+        if (id == 0) id = ++g_reqId;
+        uint8_t* f = (uint8_t*)g_ctrlTx;
+        uint32_t total = SDPCM_HDRLEN + CDC_HDRLEN + len;
+        PutSdpcm(f, total, CHAN_CONTROL);
+        uint8_t* cdc = f + SDPCM_HDRLEN;
+        Le32(cdc, cmd);
+        Le32(cdc + 4, len);
+        Le32(cdc + 8, (uint32_t)id << 16 | (set ? CDCF_SET : 0));
+        Le32(cdc + 12, 0);
+        if (len) memcpy(cdc + CDC_HDRLEN, buf, len);
+        uint32_t busLen = BusLength(total);
+        memset(f + total, 0, busLen - total);
+
+        g_respLock.Acquire();
+        g_waitId = id;
+        g_ctrlDone.store(false, std::memory_order_relaxed);
+        g_respLock.Release();
+
+        // Wait for transmit credit, pumping meanwhile: credit only ever
+        // arrives in the header of a received frame.
+        bool sent = false;
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (!sent && g_brcmf.State == BrcmfState::Running) {
+            g_brcmf.BusLock.Acquire();
+            if (CanSend()) {
+                sent = SendFrameLocked(f, busLen);
+                if (!sent) {
+                    g_brcmf.BusLock.Release();
+                    break;
+                }
+            }
+            g_brcmf.BusLock.Release();
+            if (sent) break;
+            if (Timekeeping::GetMilliseconds() - start > CREDIT_TIMEOUT_MS) break;
+            BrcmfPoll();
+            Timekeeping::BusyWaitUs(100);
+        }
+
+        bool done = false;
+        if (sent) {
+            start = Timekeeping::GetMilliseconds();
+            while (g_brcmf.State == BrcmfState::Running) {
+                if (g_ctrlDone.load(std::memory_order_acquire)) { done = true; break; }
+                if (Timekeeping::GetMilliseconds() - start > IOCTL_TIMEOUT_MS) break;
+                BrcmfPoll();
+                if (g_ctrlDone.load(std::memory_order_acquire)) { done = true; break; }
+                Timekeeping::BusyWaitUs(50);
+            }
+        }
+
+        g_respLock.Acquire();
+        g_waitId = 0;
+        bool ok = false;
+        if (done) {
+            int32_t status = g_ctrlStatus;
+            if (fwStatus) *fwStatus = status;
+            ok = !(g_ctrlFlags & CDCF_ERROR) && status >= 0;
+            if (!set && len) {
+                uint32_t n = g_ctrlRespLen < len ? g_ctrlRespLen : len;
+                memcpy(buf, g_ctrlResp, n);
+            }
+        }
+        g_respLock.Release();
+
+        if (done) {
+            if (g_brcmf.CmdFailBucket) g_brcmf.CmdFailBucket--;
+        } else if (g_brcmf.State == BrcmfState::Running) {
+            KernelLogStream(WARNING, "WiFi") << "Firmware command " << (uint64_t)cmd
+                << (sent ? " went unanswered" : " could not be sent (no transmit credit)");
+            if (++g_brcmf.CmdFailBucket >= FAIL_BUCKET_LIMIT)
+                BrcmfOnFirmwareFailure("firmware stopped answering commands");
+        }
+
+        g_ctrlLock.Release();
+        return ok;
+    }
+
+    static constexpr uint32_t WLC_GET_VAR = 262;
+    static constexpr uint32_t WLC_SET_VAR = 263;
+
+    static kcp::Mutex g_iovarLock;
+    static uint8_t g_iovarBuf[CTRL_MAX];
+
+    static uint32_t NameLen(const char* name) {
+        uint32_t n = 0;
+        while (name[n]) n++;
+        return n + 1;
+    }
+
+    bool BrcmfIovarSet(const char* name, const void* data, uint32_t len, bool quiet) {
+        uint32_t nameLen = NameLen(name);
+        if (nameLen + len > CTRL_MAX) return false;
+        g_iovarLock.Acquire();
+        memcpy(g_iovarBuf, name, nameLen);
+        if (len) memcpy(g_iovarBuf + nameLen, data, len);
+        int32_t status = 0;
+        bool ok = BrcmfIoctl(WLC_SET_VAR, true, g_iovarBuf, nameLen + len, &status);
+        g_iovarLock.Release();
+        if (!ok && !quiet)
+            KernelLogStream(WARNING, "WiFi") << "Setting \"" << name << "\" failed (status "
+                << (int)status << ")";
+        return ok;
+    }
+
+    bool BrcmfIovarGet(const char* name, void* out, uint32_t len, bool quiet) {
+        uint32_t nameLen = NameLen(name);
+        uint32_t bufLen = nameLen > len ? nameLen : len;
+        if (bufLen > CTRL_MAX) return false;
+        g_iovarLock.Acquire();
+        memset(g_iovarBuf, 0, bufLen);
+        memcpy(g_iovarBuf, name, nameLen);
+        int32_t status = 0;
+        bool ok = BrcmfIoctl(WLC_GET_VAR, false, g_iovarBuf, bufLen, &status);
+        if (ok) memcpy(out, g_iovarBuf, len);
+        g_iovarLock.Release();
+        if (!ok && !quiet)
+            KernelLogStream(WARNING, "WiFi") << "Reading \"" << name << "\" failed (status "
+                << (int)status << ")";
+        return ok;
+    }
+
+    bool BrcmfIovarSetInt(const char* name, uint32_t value, bool quiet) {
+        return BrcmfIovarSet(name, &value, 4, quiet);
+    }
+
+    bool BrcmfIovarGetInt(const char* name, uint32_t& value, bool quiet) {
+        return BrcmfIovarGet(name, &value, 4, quiet);
+    }
+
+    bool BrcmfSetInt(uint32_t cmd, uint32_t value) {
+        return BrcmfIoctl(cmd, true, &value, 4);
+    }
+
+    bool BrcmfGetInt(uint32_t cmd, uint32_t& value) {
+        uint32_t v = 0;
+        if (!BrcmfIoctl(cmd, false, &v, 4)) return false;
+        value = v;
+        return true;
+    }
+
+}

kernel/src/Drivers/Net/Wifi/BrcmfSdio.cpp

@@ -0,0 +1,436 @@
+/*
+    * BrcmfSdio.cpp
+    * SDIO host on the Raspberry Pi 4/400's Arasan SD controller
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "BrcmfSdio.hpp"
+#include <Arch/Cpu.hpp>
+#include <CppLib/Stream.hpp>
+#include <Libraries/Memory.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// Register access follows Drivers/Storage/Sdhci: the Broadcom/Arasan
+// controller only takes 32-bit accesses, so 8/16-bit fields are written by
+// read-modify-write of their dword and the transfer-mode/command pair in
+// the one write that issues the command. Linux (sdhci-iproc, bcm2835 data)
+// also waits four SD clocks after every write while the bus runs at 400 kHz
+// or below; so does this.
+//
+// SDIO commands (SDIO Simplified Specification 3.00):
+//   CMD5  IO_SEND_OP_COND, R4: OCR [23:0], ready [31]
+//   CMD52 IO_RW_DIRECT,    R5: flags [15:8], data [7:0]
+//         arg: R/W [31], function [30:28], RAW [27], address [25:9], data [7:0]
+//   CMD53 IO_RW_EXTENDED,  R5
+//         arg: R/W [31], function [30:28], block mode [27], incrementing
+//              address [26], address [25:9], count [8:0] (0 = 512)
+
+using namespace Kt;
+
+namespace Drivers::Net::Wifi::Sdio {
+
+    static constexpr uint32_t REG_BLOCK_SIZE_COUNT = 0x04;
+    static constexpr uint32_t REG_ARGUMENT         = 0x08;
+    static constexpr uint32_t REG_XFER_MODE_CMD    = 0x0C;
+    static constexpr uint32_t REG_RESPONSE         = 0x10;
+    static constexpr uint32_t REG_BUFFER_DATA      = 0x20;
+    static constexpr uint32_t REG_PRESENT_STATE    = 0x24;
+    static constexpr uint32_t REG_HOST_CONTROL     = 0x28;
+    static constexpr uint32_t REG_CLOCK_CONTROL    = 0x2C;
+    static constexpr uint32_t REG_INT_STATUS       = 0x30;
+    static constexpr uint32_t REG_INT_ENABLE       = 0x34;
+    static constexpr uint32_t REG_INT_SIGNAL       = 0x38;
+    static constexpr uint32_t REG_CAPABILITIES     = 0x40;
+    static constexpr uint32_t REG_VERSION          = 0xFC;
+
+    static constexpr uint32_t PS_CMD_INHIBIT = 1u << 0;
+    static constexpr uint32_t PS_DAT_INHIBIT = 1u << 1;
+
+    static constexpr uint32_t INT_CMD_COMPLETE  = 1u << 0;
+    static constexpr uint32_t INT_XFER_COMPLETE = 1u << 1;
+    static constexpr uint32_t INT_WRITE_READY   = 1u << 4;
+    static constexpr uint32_t INT_READ_READY    = 1u << 5;
+    static constexpr uint32_t INT_CARD          = 1u << 8;
+    static constexpr uint32_t INT_ERROR         = 1u << 15;
+    static constexpr uint32_t INT_ERROR_MASK    = 0xFFFF0000u;
+    // Status enables: the completions, both buffer-ready bits, the card
+    // interrupt and every error. Card insert/remove stay off: nothing is
+    // ever inserted, and the detect line is not wired to the radio.
+    static constexpr uint32_t INT_ENABLES = 0x03FF0000u | INT_CMD_COMPLETE | INT_XFER_COMPLETE
+                                          | INT_WRITE_READY | INT_READ_READY | INT_CARD;
+
+    static constexpr uint32_t HC_4BIT  = 1u << 1;
+    static constexpr uint32_t PWR_ON   = 1u << 8;
+    static constexpr uint32_t PWR_330  = 7u << 9;
+
+    static constexpr uint32_t CLK_INT_EN     = 1u << 0;
+    static constexpr uint32_t CLK_INT_STABLE = 1u << 1;
+    static constexpr uint32_t CLK_SD_EN      = 1u << 2;
+    static constexpr uint32_t TIMEOUT_MAX    = 0xEu << 16;
+
+    static constexpr uint32_t RESET_ALL = 1u << 24;
+    static constexpr uint32_t RESET_CMD = 1u << 25;
+    static constexpr uint32_t RESET_DAT = 1u << 26;
+
+    static constexpr uint32_t XM_BLOCK_COUNT_EN = 1u << 1;
+    static constexpr uint32_t XM_READ           = 1u << 4;
+    static constexpr uint32_t XM_MULTI_BLOCK    = 1u << 5;
+
+    enum Response : uint32_t {
+        RespNone    = 0,
+        Resp48      = 2 | 3u << 3,   // R1, R5, R6: CRC and index checked
+        Resp48NoCrc = 2,             // R4
+    };
+    static constexpr uint32_t CMD_DATA = 1u << 5;
+
+    // R5 flags: COM_CRC_ERROR, ILLEGAL_COMMAND, ERROR, FUNCTION_NUMBER,
+    // OUT_OF_RANGE.
+    static constexpr uint32_t R5_ERRORS = 0xCB00;
+
+    // CCCR
+    static constexpr uint32_t CCCR_IOEN     = 0x02;
+    static constexpr uint32_t CCCR_IORDY    = 0x03;
+    static constexpr uint32_t CCCR_INTEN    = 0x04;
+    static constexpr uint32_t CCCR_ABORT    = 0x06;
+    static constexpr uint32_t CCCR_BUSIF    = 0x07;
+    static constexpr uint32_t BUSIF_4BIT    = 0x02;
+    static constexpr uint32_t BUSIF_CD_DIS  = 0x80;
+    static constexpr uint32_t FBR_BLKSIZE   = 0x10;   // in function n's FBR at n * 0x100
+
+    static constexpr uint64_t CommandTimeoutMs = 100;
+    static constexpr uint64_t DataTimeoutMs    = 500;
+    // CMD53's count field is nine bits.
+    static constexpr uint32_t MaxBlocks = 511;
+
+    // The legacy EMMC clock the firmware sets up; only used when neither the
+    // mailbox nor the capabilities name one. Assuming it high errs towards a
+    // slower bus, never a faster one.
+    static constexpr uint32_t FallbackBaseClockHz = 250000000;
+
+    static volatile uint8_t* g_regs = nullptr;
+    static uint32_t g_version = 0;
+    static uint32_t g_baseClockHz = 0;
+    static uint32_t g_clockHz = 0;
+    static uint32_t g_rca = 0;
+    static bool g_initialized = false;
+
+    static inline uint32_t Read(uint32_t reg) {
+        return *(volatile uint32_t*)(g_regs + reg);
+    }
+
+    static inline void Write(uint32_t reg, uint32_t value) {
+        *(volatile uint32_t*)(g_regs + reg) = value;
+        if (g_clockHz <= 400000) Timekeeping::BusyWaitUs(10);
+    }
+
+    static bool WaitClear(uint32_t reg, uint32_t mask, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Read(reg) & mask) {
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
+            Arch::Pause();
+        }
+        return true;
+    }
+
+    static bool Reset(uint32_t which) {
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | which);
+        return WaitClear(REG_CLOCK_CONTROL, which, 100);
+    }
+
+    static bool WaitStatus(uint32_t bits, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            uint32_t status = Read(REG_INT_STATUS);
+            if (status & (INT_ERROR | INT_ERROR_MASK)) {
+                Write(REG_INT_STATUS, status & ~INT_CARD);
+                Reset(RESET_CMD | RESET_DAT);
+                return false;
+            }
+            if (status & bits) {
+                Write(REG_INT_STATUS, status & bits);
+                return true;
+            }
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) {
+                Reset(RESET_CMD | RESET_DAT);
+                return false;
+            }
+            Arch::Pause();
+        }
+    }
+
+    static bool SendCommand(uint32_t index, uint32_t argument, uint32_t response,
+                            uint32_t transferMode = 0, bool data = false) {
+        uint32_t inhibit = PS_CMD_INHIBIT | (data ? PS_DAT_INHIBIT : 0);
+        if (!WaitClear(REG_PRESENT_STATE, inhibit, CommandTimeoutMs)) {
+            Reset(RESET_CMD | RESET_DAT);
+            return false;
+        }
+
+        Write(REG_INT_STATUS, ~INT_CARD);
+        Write(REG_ARGUMENT, argument);
+        uint32_t command = index << 8 | response | (data ? CMD_DATA : 0);
+        Write(REG_XFER_MODE_CMD, command << 16 | transferMode);
+        return WaitStatus(INT_CMD_COMPLETE, CommandTimeoutMs);
+    }
+
+    static bool SetClock(uint32_t hz) {
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) & ~0xFFFFu);
+
+        uint32_t divider = 0;   // base / (2 * divider); 0 = undivided
+        if (hz < g_baseClockHz) {
+            divider = (g_baseClockHz + 2 * hz - 1) / (2 * hz);
+            uint32_t max = g_version >= 2 ? 0x3FF : 0x80;
+            if (divider > max) divider = max;
+            if (g_version < 2) {
+                uint32_t p = 1;
+                while (p < divider) p <<= 1;
+                divider = p;
+            }
+        }
+        uint32_t actual = divider ? g_baseClockHz / (2 * divider) : g_baseClockHz;
+
+        uint32_t field = (divider & 0xFF) << 8 | ((divider >> 8) & 0x3) << 6;
+        Write(REG_CLOCK_CONTROL, (Read(REG_CLOCK_CONTROL) & ~0xFFFFu) | field | CLK_INT_EN);
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (!(Read(REG_CLOCK_CONTROL) & CLK_INT_STABLE)) {
+            if (Timekeeping::GetMilliseconds() - start > 20) return false;
+            Arch::Pause();
+        }
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | CLK_SD_EN);
+        g_clockHz = actual;
+        return true;
+    }
+
+    static bool Cmd52(bool write, uint32_t fn, uint32_t addr, uint8_t in, uint8_t* out) {
+        uint32_t arg = (write ? 1u << 31 : 0) | (fn & 7) << 28 | (addr & 0x1FFFF) << 9 | in;
+        if (!SendCommand(52, arg, Resp48)) return false;
+        uint32_t r = Read(REG_RESPONSE);
+        if (r & R5_ERRORS) return false;
+        if (out) *out = (uint8_t)r;
+        return true;
+    }
+
+    bool Read8(uint32_t fn, uint32_t addr, uint8_t& value) {
+        return Cmd52(false, fn, addr, 0, &value);
+    }
+
+    bool Write8(uint32_t fn, uint32_t addr, uint8_t value) {
+        return Cmd52(true, fn, addr, value, nullptr);
+    }
+
+    // One CMD53: `count` blocks of `blockSize` bytes, or (byte mode) one run
+    // of `blockSize` bytes. `words` is 4-byte aligned.
+    static bool Cmd53(bool write, uint32_t fn, uint32_t addr, bool incr, uint32_t* words,
+                      bool blockMode, uint32_t count, uint32_t blockSize) {
+        uint32_t arg = (write ? 1u << 31 : 0) | (fn & 7) << 28 | (blockMode ? 1u << 27 : 0)
+                     | (incr ? 1u << 26 : 0) | (addr & 0x1FFFF) << 9
+                     | ((blockMode ? count : blockSize) & 0x1FF);
+        uint32_t blocks = blockMode ? count : 1;
+        Write(REG_BLOCK_SIZE_COUNT, blocks << 16 | blockSize);
+
+        uint32_t mode = write ? 0 : XM_READ;
+        if (blockMode) mode |= XM_BLOCK_COUNT_EN | (blocks > 1 ? XM_MULTI_BLOCK : 0);
+        if (!SendCommand(53, arg, Resp48, mode, true)) return false;
+        if (Read(REG_RESPONSE) & R5_ERRORS) {
+            Abort(fn);
+            Reset(RESET_CMD | RESET_DAT);
+            return false;
+        }
+
+        uint32_t perBlock = (blockSize + 3) / 4;
+        for (uint32_t b = 0; b < blocks; b++) {
+            if (!WaitStatus(write ? INT_WRITE_READY : INT_READ_READY, DataTimeoutMs)) {
+                Abort(fn);
+                return false;
+            }
+            for (uint32_t i = 0; i < perBlock; i++) {
+                if (write) *(volatile uint32_t*)(g_regs + REG_BUFFER_DATA) = *words++;
+                else *words++ = Read(REG_BUFFER_DATA);
+            }
+        }
+        if (!WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs)) {
+            Abort(fn);
+            return false;
+        }
+        return true;
+    }
+
+    static bool Transfer(bool write, uint32_t fn, uint32_t addr, bool incr,
+                         uint8_t* buffer, uint32_t len) {
+        if (!g_initialized || (len & 3) != 0) return false;
+        uint32_t blockSize = fn == FN2 ? FN2_BLOCK : FN1_BLOCK;
+
+        // The data port moves dwords; callers' buffers are only byte aligned
+        // in general.
+        uint32_t bounce[FN2_BLOCK / 4];
+        while (len > 0) {
+            bool blockMode = len >= blockSize;
+            uint32_t count = blockMode ? len / blockSize : 0;
+            if (count > MaxBlocks) count = MaxBlocks;
+            uint32_t run = blockMode ? count * blockSize : len;
+            if (!blockMode && run > FN2_BLOCK) run = FN2_BLOCK;
+
+            bool aligned = ((uintptr_t)buffer & 3) == 0;
+            if (aligned) {
+                if (!Cmd53(write, fn, addr, incr, (uint32_t*)buffer, blockMode, count,
+                           blockMode ? blockSize : run)) return false;
+            } else {
+                // Unaligned: one block (or the byte-mode run) at a time
+                // through the bounce buffer.
+                if (blockMode) { count = 1; run = blockSize; }
+                if (write) memcpy(bounce, buffer, run);
+                if (!Cmd53(write, fn, addr, incr, bounce, blockMode, count,
+                           blockMode ? blockSize : run)) return false;
+                if (!write) memcpy(buffer, bounce, run);
+            }
+            buffer += run;
+            if (incr) addr += run;
+            len -= run;
+        }
+        return true;
+    }
+
+    bool ReadBlock(uint32_t fn, uint32_t addr, bool incr, uint8_t* buffer, uint32_t len) {
+        return Transfer(false, fn, addr, incr, buffer, len);
+    }
+
+    bool WriteBlock(uint32_t fn, uint32_t addr, bool incr, const uint8_t* buffer, uint32_t len) {
+        return Transfer(true, fn, addr, incr, (uint8_t*)buffer, len);
+    }
+
+    void Abort(uint32_t fn) {
+        Cmd52(true, FN0, CCCR_ABORT, (uint8_t)fn, nullptr);
+    }
+
+    bool EnableFunction(uint32_t fn, bool enable, uint64_t timeoutMs) {
+        uint8_t ioen = 0;
+        if (!Read8(FN0, CCCR_IOEN, ioen)) return false;
+        uint8_t bit = (uint8_t)(1u << fn);
+        ioen = enable ? (ioen | bit) : (ioen & ~bit);
+        if (!Write8(FN0, CCCR_IOEN, ioen)) return false;
+        if (!enable) return true;
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            uint8_t ready = 0;
+            if (Read8(FN0, CCCR_IORDY, ready) && (ready & bit)) return true;
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
+            Timekeeping::BusyWaitUs(500);
+        }
+    }
+
+    bool EnableCardInterrupts(uint8_t mask) {
+        return Write8(FN0, CCCR_INTEN, mask);
+    }
+
+    bool CardInterruptPending() {
+        return g_initialized && (Read(REG_INT_STATUS) & INT_CARD) != 0;
+    }
+
+    bool IsInitialized() { return g_initialized; }
+
+    bool HostInit(uint64_t mmioPhys, uint32_t baseClockHz) {
+        g_initialized = false;
+        if (!g_regs) {
+            Memory::VMM::g_paging->MapMMIO(mmioPhys & ~0xFFFULL, Memory::HHDM(mmioPhys & ~0xFFFULL));
+            g_regs = (volatile uint8_t*)Memory::HHDM(mmioPhys);
+        }
+        g_clockHz = 0;
+
+        if (!Reset(RESET_ALL)) {
+            KernelLogStream(ERROR, "WiFi-SDIO") << "Controller reset timed out";
+            return false;
+        }
+
+        g_version = (Read(REG_VERSION) >> 16) & 0xFF;
+        uint32_t caps = Read(REG_CAPABILITIES);
+        uint32_t capsMhz = g_version >= 2 ? (caps >> 8) & 0xFF : (caps >> 8) & 0x3F;
+        g_baseClockHz = baseClockHz ? baseClockHz
+                      : capsMhz ? capsMhz * 1000000 : FallbackBaseClockHz;
+
+        // The radio's 3.3 V supply is not switched by the controller, but the
+        // bus power bit gates the controller's own drivers.
+        Write(REG_HOST_CONTROL, (Read(REG_HOST_CONTROL) & ~0xFF00u) | PWR_330 | PWR_ON);
+        Write(REG_INT_ENABLE, INT_ENABLES);
+        Write(REG_INT_SIGNAL, 0);
+        Write(REG_CLOCK_CONTROL, (Read(REG_CLOCK_CONTROL) & ~0xFF0000u) | TIMEOUT_MAX);
+
+        if (!SetClock(400000)) {
+            KernelLogStream(ERROR, "WiFi-SDIO") << "Clock did not stabilize";
+            return false;
+        }
+        // 74 clocks before the first command.
+        Timekeeping::BusyWaitMs(2);
+
+        KernelLogStream(INFO, "WiFi-SDIO") << "Arasan controller at 0x" << base::hex << mmioPhys
+            << base::dec << ", version " << (uint64_t)(g_version + 1) << ".00, base clock "
+            << (uint64_t)(g_baseClockHz / 1000000) << " MHz";
+        return true;
+    }
+
+    bool CardInit() {
+        // Allow Transfer()/Cmd52 during bring-up; cleared again on failure.
+        g_initialized = true;
+        auto fail = [](const char* what) {
+            KernelLogStream(WARNING, "WiFi-SDIO") << what;
+            g_initialized = false;
+            return false;
+        };
+
+        SendCommand(0, 0, RespNone);
+        Timekeeping::BusyWaitMs(2);
+
+        // CMD5 with no voltage first to read the OCR; the radio may still be
+        // coming out of reset, so give it a few tries.
+        uint32_t ocr = 0;
+        bool answered = false;
+        for (int i = 0; i < 10 && !answered; i++) {
+            answered = SendCommand(5, 0, Resp48NoCrc);
+            if (!answered) Timekeeping::BusyWaitMs(10);
+        }
+        if (!answered) return fail("No SDIO card answers CMD5");
+        ocr = Read(REG_RESPONSE) & 0x00FFFFFF;
+
+        uint32_t want = ocr & 0x00300000;   // 3.2-3.4 V
+        if (!want) want = ocr & 0x00FF8000;
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            if (!SendCommand(5, want, Resp48NoCrc)) return fail("CMD5 failed");
+            if (Read(REG_RESPONSE) & (1u << 31)) break;
+            if (Timekeeping::GetMilliseconds() - start > 1000) return fail("Card stayed busy (CMD5)");
+            Timekeeping::BusyWaitMs(1);
+        }
+
+        if (!SendCommand(3, 0, Resp48)) return fail("CMD3 failed");
+        g_rca = Read(REG_RESPONSE) >> 16;
+        if (!SendCommand(7, g_rca << 16, Resp48)) return fail("CMD7 failed");
+
+        // 4-bit bus; disconnect the card-detect pull-up on DAT3.
+        uint8_t busif = 0;
+        if (!Read8(FN0, CCCR_BUSIF, busif)) return fail("CCCR read failed");
+        busif = (uint8_t)((busif & ~0x03) | BUSIF_4BIT | BUSIF_CD_DIS);
+        if (!Write8(FN0, CCCR_BUSIF, busif)) return fail("Could not select the 4-bit bus");
+        Write(REG_HOST_CONTROL, Read(REG_HOST_CONTROL) | HC_4BIT);
+
+        if (!SetClock(25000000)) return fail("Clock did not stabilize at 25 MHz");
+
+        // Function block sizes (little endian in the FBR).
+        bool ok = Write8(FN0, 1 * 0x100 + FBR_BLKSIZE, (uint8_t)(FN1_BLOCK & 0xFF))
+               && Write8(FN0, 1 * 0x100 + FBR_BLKSIZE + 1, (uint8_t)(FN1_BLOCK >> 8))
+               && Write8(FN0, 2 * 0x100 + FBR_BLKSIZE, (uint8_t)(FN2_BLOCK & 0xFF))
+               && Write8(FN0, 2 * 0x100 + FBR_BLKSIZE + 1, (uint8_t)(FN2_BLOCK >> 8));
+        if (!ok) return fail("Could not set the function block sizes");
+
+        if (!EnableFunction(FN1, true, 1000)) return fail("Function 1 did not become ready");
+
+        KernelLogStream(OK, "WiFi-SDIO") << "SDIO card ready (RCA 0x" << base::hex << (uint64_t)g_rca
+            << base::dec << ", OCR 0x" << base::hex << (uint64_t)ocr << base::dec << ", "
+            << (uint64_t)(g_clockHz / 1000) << " kHz, 4-bit)";
+        return true;
+    }
+
+}

kernel/src/Drivers/Net/Wifi/BrcmfSdio.hpp

@@ -0,0 +1,63 @@
+/*
+    * BrcmfSdio.hpp
+    * SDIO host on the Raspberry Pi 4/400's Arasan SD controller, for the
+    * CYW4345x Wi-Fi radio wired to it
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// The BCM2711 has two SD controllers: EMMC2 runs the card slot
+// (Drivers/Storage/Sdhci) and the older Arasan-derived EMMC, routed to GPIO
+// 34-39, runs the radio's SDIO interface. This is a minimal SDIO host for
+// the latter: one card, polled, programmed I/O, CMD52 and CMD53 only.
+//
+// None of these calls lock; the Broadcom driver above serializes them.
+
+namespace Drivers::Net::Wifi::Sdio {
+
+    // SDIO functions on the radio.
+    constexpr uint32_t FN0 = 0;   // CCCR / FBR
+    constexpr uint32_t FN1 = 1;   // backplane
+    constexpr uint32_t FN2 = 2;   // WLAN frames
+
+    // Block sizes the card is configured with.
+    constexpr uint32_t FN1_BLOCK = 64;
+    constexpr uint32_t FN2_BLOCK = 512;
+
+    // Map the controller's registers and reset it. `baseClockHz` is the
+    // controller's input clock (0 = take it from the capabilities).
+    bool HostInit(uint64_t mmioPhys, uint32_t baseClockHz);
+
+    // Enumerate the card: CMD0/5/3/7, 4-bit bus at 25 MHz, function 1/2
+    // block sizes, function 1 enabled. Returns false when nothing answers.
+    bool CardInit();
+
+    // CMD52: one byte.
+    bool Read8(uint32_t fn, uint32_t addr, uint8_t& value);
+    bool Write8(uint32_t fn, uint32_t addr, uint8_t value);
+
+    // CMD53: `len` bytes, a multiple of 4. `incr` selects an incrementing
+    // address (backplane memory) over a fixed one (the function 2 FIFO).
+    // Longer transfers are split into block-mode runs plus a byte-mode tail.
+    bool ReadBlock(uint32_t fn, uint32_t addr, bool incr, uint8_t* buffer, uint32_t len);
+    bool WriteBlock(uint32_t fn, uint32_t addr, bool incr, const uint8_t* buffer, uint32_t len);
+
+    // Enable or disable function `fn`'s I/O; waits up to `timeoutMs` for the
+    // card to report it ready.
+    bool EnableFunction(uint32_t fn, bool enable, uint64_t timeoutMs);
+
+    // CCCR interrupt enables (master | functions).
+    bool EnableCardInterrupts(uint8_t mask);
+
+    // Abort a function's transfer in progress (CCCR I/O Abort).
+    void Abort(uint32_t fn);
+
+    // The card is signalling an interrupt on DAT1. One register read, safe
+    // from any context.
+    bool CardInterruptPending();
+
+    bool IsInitialized();
+
+}

kernel/src/Drivers/Net/Wifi/Iwx.hpp

@@ -443,6 +443,12 @@ namespace Drivers::Net::Wifi {
     // Send an Ethernet frame over the air (called by the netif).
     bool IwxConnectSendEthernet(const uint8_t* frame, uint32_t len);
 
+    // The supplicant's hooks (Wpa.hpp) for this MLME; Wifi.cpp routes them.
+    bool IwxWpaTxEapol(const uint8_t* body, uint32_t len);
+    bool IwxWpaInstallPtk(const uint8_t* tk, uint32_t tkLen, uint8_t cipher);
+    bool IwxWpaInstallGtk(const uint8_t* gtk, uint32_t gtkLen, uint8_t keyIdx,
+                          uint8_t cipher, const uint8_t* rsc);
+
     // Sink for decapsulated Ethernet frames, implemented by Wifi.cpp.
     void WifiRxEthernet(const uint8_t* frame, uint32_t len);
 }

kernel/src/Drivers/Net/Wifi/IwxConnect.cpp

@@ -535,7 +535,7 @@ namespace Drivers::Net::Wifi {
 
     // The supplicant transmits EAPOL frames unencrypted: they are what
     // establishes the key in the first place.
-    bool WpaTxEapol(const uint8_t* body, uint32_t len) {
+    bool IwxWpaTxEapol(const uint8_t* body, uint32_t len) {
         return SendDataFrame(g_bssid, ETHERTYPE_EAPOL, body, len, false);
     }
 
@@ -545,13 +545,13 @@ namespace Drivers::Net::Wifi {
     static InstalledKey g_ptkKey = {};
     static InstalledKey g_gtkKey = {};
 
-    bool WpaInstallPtk(const uint8_t* tk, uint32_t tkLen, uint8_t cipher) {
+    bool IwxWpaInstallPtk(const uint8_t* tk, uint32_t tkLen, uint8_t cipher) {
         if (!IwxSetKey(tk, tkLen, 0, true, cipher, nullptr)) return false;
         g_ptkKey = { true, 0, cipher, (uint8_t)tkLen };
         return true;
     }
 
-    bool WpaInstallGtk(const uint8_t* gtk, uint32_t gtkLen, uint8_t keyIdx,
+    bool IwxWpaInstallGtk(const uint8_t* gtk, uint32_t gtkLen, uint8_t keyIdx,
                           uint8_t cipher, const uint8_t* rsc) {
         if (gtkLen != 16 && gtkLen != 32) return false;
         if (!IwxSetKey(gtk, gtkLen, keyIdx, false, cipher, rsc)) return false;

kernel/src/Drivers/Net/Wifi/Wifi.cpp

@@ -14,6 +14,7 @@
 #include "Wifi.hpp"
 #include <Arch/Cpu.hpp>
 #include "Iwx.hpp"
+#include "Brcmf.hpp"
 #include "Wpa.hpp"
 #include <Fs/Vfs.hpp>
 #include <Libraries/Memory.hpp>
@@ -34,6 +35,72 @@ namespace Drivers::Net::Wifi {
     static std::atomic<bool> g_initPending{false};
     static bool g_initialized = false;
 
+    // =========================================================================
+    // Backend
+    //
+    // Two kinds of adapter sit behind this facade: the Intel AX210 family on
+    // PCIe (Iwx*: the firmware is a SoftMAC, the MLME is ours) and the
+    // Raspberry Pi 4/400's Broadcom/Cypress radio on SDIO (Brcmf*: FullMAC,
+    // the firmware joins by itself). A machine has at most one; whichever
+    // probes first owns the facade. Both use the same scan table, the same
+    // supplicant and the same connection-state numbering (WIFI_CONN_*, which
+    // IwxConnStateId mirrors).
+    // =========================================================================
+
+    enum class Backend : uint8_t { None, Iwx, Brcmf };
+    static Backend g_backend = Backend::None;
+
+    static bool IsBrcmf() { return g_backend == Backend::Brcmf; }
+
+    static bool FwRunning() {
+        return IsBrcmf() ? BrcmfGetState() == BrcmfState::Running
+                         : g_iwx.State == IwxFwState::Running;
+    }
+
+    static bool FwFailed() {
+        return IsBrcmf() ? BrcmfGetState() == BrcmfState::Error
+                         : g_iwx.State == IwxFwState::Error;
+    }
+
+    static bool ScanActive() { return IsBrcmf() ? BrcmfScanActive() : g_iwx.ScanActive; }
+    static bool BeginScan() { return IsBrcmf() ? BrcmfStartScan() : IwxStartScan(nullptr); }
+
+    static void AbortScan() {
+        if (IsBrcmf()) BrcmfAbortScan();
+        else IwxAbortScan();
+    }
+
+    // What a blocking wait pumps: the receive path, and for a join the
+    // state machine behind it.
+    static void PumpEvents() {
+        if (IsBrcmf()) BrcmfService();
+        else IwxProcessEvents();
+    }
+
+    static void PumpConnection() {
+        if (IsBrcmf()) {
+            BrcmfService();
+        } else {
+            IwxProcessEvents();
+            IwxConnectService();
+        }
+    }
+
+    static int ConnState() { return IsBrcmf() ? BrcmfConnectState() : IwxConnectState(); }
+
+    static void ConnAbort() {
+        if (IsBrcmf()) BrcmfConnectAbort();
+        else IwxConnectAbort();
+    }
+
+    // A handshake that got as far as exchanging EAPOL frames and then failed
+    // is almost always a wrong passphrase.
+    static bool JoinFailedOnAuth() {
+        return IsBrcmf() ? BrcmfJoinFailedOnAuth() : WpaGetState() == WpaState::Failed;
+    }
+
+    static bool LinkUp() { return IsBrcmf() ? BrcmfLinkUp() : IwxLinkUp(); }
+
     // =========================================================================
     // Scan result table
     // =========================================================================
@@ -141,22 +208,12 @@ namespace Drivers::Net::Wifi {
         uint8_t RsnLen = 0;
     };
 
-    static bool ParseBeacon(const uint8_t* frame, uint32_t len, ParsedBeacon* out) {
-        // 24-byte MAC header, then timestamp(8) + beacon interval(2) + caps(2).
-        constexpr uint32_t HDR = 24;
-        constexpr uint32_t FIXED = 12;
-        if (len < HDR + FIXED) return false;
-
-        const uint8_t* fixed = frame + HDR;
-        out->BeaconInterval = (uint16_t)(fixed[8] | (fixed[9] << 8));
-        uint16_t caps = (uint16_t)(fixed[10] | (fixed[11] << 8));
-        bool privacy = (caps & 0x0010) != 0;
-
+    // The information elements of a beacon or probe response; `privacy` is
+    // the capability field's Privacy bit.
+    static void ParseIes(const uint8_t* ie, const uint8_t* end, bool privacy, ParsedBeacon* out) {
         bool haveRsn = false, haveWpa = false;
 
-        const uint8_t* ie = frame + HDR + FIXED;
-        const uint8_t* end = frame + len;
-        while (ie + 2 <= end) {
+        while (ie && ie + 2 <= end) {
             uint8_t id = ie[0];
             uint8_t ielen = ie[1];
             if (ie + 2 + ielen > end) break;
@@ -200,9 +257,24 @@ namespace Drivers::Net::Wifi {
             else if (privacy) out->Security = WIFI_SEC_WEP;
             else out->Security = WIFI_SEC_OPEN;
         }
+    }
+
+    static bool ParseBeacon(const uint8_t* frame, uint32_t len, ParsedBeacon* out) {
+        // 24-byte MAC header, then timestamp(8) + beacon interval(2) + caps(2).
+        constexpr uint32_t HDR = 24;
+        constexpr uint32_t FIXED = 12;
+        if (len < HDR + FIXED) return false;
+
+        const uint8_t* fixed = frame + HDR;
+        out->BeaconInterval = (uint16_t)(fixed[8] | (fixed[9] << 8));
+        uint16_t caps = (uint16_t)(fixed[10] | (fixed[11] << 8));
+        ParseIes(frame + HDR + FIXED, frame + len, (caps & 0x0010) != 0, out);
         return true;
     }
 
+    static void StoreScanResult(const uint8_t* bssid, const ParsedBeacon& pb, uint8_t ch,
+                                int8_t rssiDbm);
+
     // Called by the MVM RX path for every received management frame.
     void WifiRxMgmtFrame(const uint8_t* frame, uint32_t len, uint8_t channel,
                          int8_t rssiDbm) {
@@ -224,9 +296,21 @@ namespace Drivers::Net::Wifi {
         ParsedBeacon pb;
         if (!ParseBeacon(frame, len, &pb)) return;
 
-        const uint8_t* bssid = frame + 16;           // addr3 of a beacon
-        uint8_t ch = pb.Channel ? pb.Channel : channel;
+        // addr3 of a beacon
+        StoreScanResult(frame + 16, pb, pb.Channel ? pb.Channel : channel, rssiDbm);
+    }
 
+    // One BSS a FullMAC firmware reported from its own scan.
+    void WifiRxBssInfo(const uint8_t* bssid, uint16_t beaconInterval, uint16_t capability,
+                       const uint8_t* ies, uint32_t iesLen, uint8_t channel, int8_t rssiDbm) {
+        ParsedBeacon pb;
+        pb.BeaconInterval = beaconInterval;
+        ParseIes(ies, ies ? ies + iesLen : nullptr, (capability & 0x0010) != 0, &pb);
+        StoreScanResult(bssid, pb, pb.Channel ? pb.Channel : channel, rssiDbm);
+    }
+
+    static void StoreScanResult(const uint8_t* bssid, const ParsedBeacon& pb, uint8_t ch,
+                                int8_t rssiDbm) {
         g_resultLock.Acquire();
 
         ScanEntry* slot = nullptr;
@@ -275,7 +359,9 @@ namespace Drivers::Net::Wifi {
     // =========================================================================
 
     bool Probe(const Pci::PciDevice& dev) {
+        if (g_backend != Backend::None) return false;
         if (!IwxProbe(dev)) return false;
+        g_backend = Backend::Iwx;
 
         // Firmware comes off the ramdisk, which is not mounted during the PCI
         // scan; hand the rest of the bring-up to the idle loop.
@@ -283,7 +369,23 @@ namespace Drivers::Net::Wifi {
         return true;
     }
 
+    bool ProbeBroadcomSdio(uint64_t mmioPhys, const char* name) {
+        if (g_backend != Backend::None) return false;
+        if (!BrcmfProbe(mmioPhys)) return false;
+        g_backend = Backend::Brcmf;
+        KernelLogStream(INFO, "WiFi") << "Broadcom SDIO radio on " << name
+            << "; firmware loads once the root filesystem is up";
+        // Same reason as the PCI path: the firmware is on the root volume.
+        g_initPending.store(true, std::memory_order_release);
+        return true;
+    }
+
     static void CompleteInit() {
+        if (IsBrcmf()) {
+            g_initialized = BrcmfCompleteInit();
+            return;
+        }
+
         if (!IwxReadFirmware()) {
             g_iwx.State = IwxFwState::Error;
             return;
@@ -348,7 +450,7 @@ namespace Drivers::Net::Wifi {
     static constexpr uint64_t RECOVERY_BACKOFF_MS = 5000;
 
     static void ServiceRecovery() {
-        if (g_iwx.State != IwxFwState::Error || g_recoveryGaveUp) return;
+        if (!FwFailed() || g_recoveryGaveUp) return;
         // An adapter that never finished its first bring-up is ServiceDeferredInit's
         // problem, not this one.
         if (!g_initialized) return;
@@ -366,7 +468,7 @@ namespace Drivers::Net::Wifi {
 
         // Re-check under the gate: the winner of the race may have just
         // finished a reset that fixed things.
-        if (g_iwx.State != IwxFwState::Error) return;
+        if (!FwFailed()) return;
 
         uint64_t now = Timekeeping::GetMilliseconds();
         if (g_lastRecoveryMs && now - g_lastRecoveryMs < RECOVERY_BACKOFF_MS) return;
@@ -384,6 +486,24 @@ namespace Drivers::Net::Wifi {
             << " (attempt " << (uint64_t)g_recoveryAttempts << " of "
             << (uint64_t)MAX_RECOVERY_ATTEMPTS << ")";
 
+        if (IsBrcmf()) {
+            // A FullMAC radio is reset by power: WL_ON low, then the whole
+            // bring-up (firmware download included) again.
+            g_initialized = false;
+            g_asyncConnect = false;
+            g_scanDeadline = 0;
+            g_lastError = WIFI_ERR_NO_ADAPTER;
+            ClearResults();
+            BrcmfShutdown();
+            CompleteInit();
+            if (g_initialized) {
+                KernelLogStream(OK, "WiFi") << "Adapter recovered";
+                g_recoveryAttempts = 0;
+                g_lastError = 0;
+            }
+            return;
+        }
+
         // Nothing may be mid-command while the device is torn down.  Taking
         // CmdLock is enough: it is what serializes every sender, and the RX
         // pump has its own reentrancy guard.
@@ -517,8 +637,24 @@ namespace Drivers::Net::Wifi {
         return false;
     }
 
+    // The facade's own deadlines (async scan and join) and a due reset.
+    static bool FacadeWorkDue(uint64_t now) {
+        if (g_scanDeadline != 0 && (now >= g_scanDeadline || !ScanActive())) return true;
+        if (g_asyncConnect) return true;
+        return FwFailed() && g_initialized && !g_recoveryGaveUp &&
+               (g_lastRecoveryMs == 0 || now - g_lastRecoveryMs >= RECOVERY_BACKOFF_MS);
+    }
+
+    bool HasPolledWork() {
+        return IsBrcmf() && BrcmfHasWork(Timekeeping::GetMilliseconds());
+    }
+
     bool HasDeferredWork() {
         if (InitDue()) return true;
+        if (IsBrcmf()) {
+            uint64_t now = Timekeeping::GetMilliseconds();
+            return BrcmfHasWork(now) || FacadeWorkDue(now);
+        }
         if (!g_iwx.Mmio) return false;
         if (g_iwx.WorkPending) return true;
 
@@ -527,7 +663,29 @@ namespace Drivers::Net::Wifi {
         return HasBlockingWork(now);
     }
 
+    // The Broadcom radio has no interrupt in use: everything on its bus is
+    // polled, and everything on its bus waits (SDIO transfers, ioctls), so
+    // all of it goes through the blocking gate.
+    static void ServiceBrcmf() {
+        uint64_t now = Timekeeping::GetMilliseconds();
+        bool radioWork = BrcmfHasWork(now);
+        if (!radioWork && !FacadeWorkDue(now)) return;
+        if (LeaveToAp()) return;
+        if (__atomic_test_and_set(&g_blockingBusy, __ATOMIC_ACQUIRE)) return;
+        struct Guard {
+            ~Guard() { __atomic_clear(&g_blockingBusy, __ATOMIC_RELEASE); }
+        } guard;
+
+        if (radioWork) BrcmfService();
+        ServiceAsync();
+        ServiceRecovery();
+    }
+
     void ServiceEvents() {
+        if (IsBrcmf()) {
+            ServiceBrcmf();
+            return;
+        }
         if (!g_iwx.Mmio) return;
 
         // The non-blocking half: any core, the BSP included.
@@ -573,7 +731,11 @@ namespace Drivers::Net::Wifi {
     }
 
     bool IsInitialized() { return g_initialized; }
-    bool IsPresent() { return g_iwx.State != IwxFwState::Absent; }
+
+    bool IsPresent() {
+        return IsBrcmf() ? BrcmfGetState() != BrcmfState::Absent
+                         : g_iwx.State != IwxFwState::Absent;
+    }
 
     // =========================================================================
     // Public operations
@@ -611,26 +773,26 @@ namespace Drivers::Net::Wifi {
     int Scan(WifiNetwork* out, int maxCount, uint32_t timeoutMs) {
         if (!out || maxCount <= 0) return -1;
         if (!g_initialized) return -1;
-        if (g_iwx.State != IwxFwState::Running) return -1;
+        if (!FwRunning()) return -1;
 
         if (timeoutMs < 1000) timeoutMs = 1000;
         if (timeoutMs > 20000) timeoutMs = 20000;
 
         ClearResults();
 
-        if (!IwxStartScan(nullptr)) return -1;
+        if (!BeginScan()) return -1;
 
         uint64_t start = Timekeeping::GetMilliseconds();
-        while (g_iwx.ScanActive && Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            IwxProcessEvents();
+        while (ScanActive() && Timekeeping::GetMilliseconds() - start < timeoutMs) {
+            PumpEvents();
             for (int i = 0; i < 200; i++) Arch::Pause();
         }
 
-        if (g_iwx.ScanActive) {
-            IwxAbortScan();
+        if (ScanActive()) {
+            AbortScan();
             // Drain whatever the firmware queued before the abort landed.
             uint64_t t0 = Timekeeping::GetMilliseconds();
-            while (Timekeeping::GetMilliseconds() - t0 < 200) IwxProcessEvents();
+            while (Timekeeping::GetMilliseconds() - t0 < 200) PumpEvents();
         }
 
         g_scanGeneration++;
@@ -639,14 +801,14 @@ namespace Drivers::Net::Wifi {
 
     int StartScan(uint32_t timeoutMs) {
         if (!g_initialized) return -1;
-        if (g_iwx.State != IwxFwState::Running) return -1;
-        if (g_iwx.ScanActive) return 1;              // already sweeping
+        if (!FwRunning()) return -1;
+        if (ScanActive()) return 1;                  // already sweeping
 
         if (timeoutMs < 1000) timeoutMs = 1000;
         if (timeoutMs > 20000) timeoutMs = 20000;
 
         ClearResults();
-        if (!IwxStartScan(nullptr)) return -1;
+        if (!BeginScan()) return -1;
 
         g_scanDeadline = Timekeeping::GetMilliseconds() + timeoutMs;
         return 0;
@@ -657,33 +819,47 @@ namespace Drivers::Net::Wifi {
         memset(out, 0, sizeof(*out));
 
         out->present = IsPresent() ? 1 : 0;
-        out->state = (uint8_t)g_iwx.State;
         if (!IsPresent()) return -1;
 
+        const char* fwVersion;
+        if (IsBrcmf()) {
+            out->state = (uint8_t)BrcmfGetState();
+            memcpy(out->mac, g_brcmf.Mac, 6);
+            out->bands = (uint8_t)((g_brcmf.Band24 ? 1 : 0) | (g_brcmf.Band5 ? 2 : 0));
+            out->channels = g_brcmf.ChannelCount;
+            out->rxPackets = g_brcmf.RxPackets;
+            out->txPackets = g_brcmf.TxPackets;
+            out->fwErrors = g_brcmf.FwErrors;
+            fwVersion = g_brcmf.FwVersion;
+        } else {
+            out->state = (uint8_t)g_iwx.State;
             memcpy(out->mac, g_iwx.Nvm.HwAddr, 6);
-        out->scanning = g_iwx.ScanActive ? 1 : 0;
             out->bands = (uint8_t)((g_iwx.Nvm.Sku24GHz ? 1 : 0)
                                  | (g_iwx.Nvm.Sku52GHz ? 2 : 0));
             out->channels = (uint16_t)g_iwx.ChannelCount;
             out->rxPackets = g_iwx.RxPackets;
             out->txPackets = g_iwx.TxPackets;
             out->fwErrors = (uint32_t)g_iwx.FwErrors;
-        out->connState = (uint32_t)IwxConnectState();
-        out->connected = IwxLinkUp() ? 1 : 0;
+            fwVersion = g_iwx.Fw.Version;
+        }
+        out->scanning = ScanActive() ? 1 : 0;
+        out->connState = (uint32_t)ConnState();
+        out->connected = LinkUp() ? 1 : 0;
         out->lastError = (int32_t)g_lastError;
         out->scanGeneration = g_scanGeneration;
         out->joining = g_asyncConnect ? 1 : 0;
 
-        if (IwxConnectState() != (int)IwxConnStateId::Idle) {
-            const char* ssid = IwxConnectSsid();
+        if (ConnState() != (int)IwxConnStateId::Idle) {
+            const char* ssid = IsBrcmf() ? BrcmfConnectSsid() : IwxConnectSsid();
             int k = 0;
             for (; k < 32 && ssid[k]; k++) out->ssid[k] = ssid[k];
             out->ssid[k] = '\0';
-            memcpy(out->bssid, IwxConnectBssid(), 6);
+            memcpy(out->bssid, IsBrcmf() ? BrcmfConnectBssid() : IwxConnectBssid(), 6);
+            if (IsBrcmf()) out->channel = BrcmfConnectChannel();
         }
 
         int i = 0;
-        for (; i < 31 && g_iwx.Fw.Version[i]; i++) out->fwVersion[i] = g_iwx.Fw.Version[i];
+        for (; i < 31 && fwVersion[i]; i++) out->fwVersion[i] = fwVersion[i];
         out->fwVersion[i] = '\0';
         return 0;
     }
@@ -695,24 +871,20 @@ namespace Drivers::Net::Wifi {
         uint64_t start = Timekeeping::GetMilliseconds();
 
         while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
-            IwxProcessEvents();
-            IwxConnectService();
+            PumpConnection();
 
-            auto state = (IwxConnStateId)IwxConnectState();
+            auto state = (IwxConnStateId)ConnState();
             if (state == IwxConnStateId::Connected) return 0;
             if (state == IwxConnStateId::Failed) {
-                // A handshake that got as far as exchanging EAPOL frames and
-                // then failed is almost always a wrong passphrase.
-                int rc = WpaGetState() == WpaState::Failed
-                         ? WIFI_ERR_AUTH : WIFI_ERR_FAILED;
+                int rc = JoinFailedOnAuth() ? WIFI_ERR_AUTH : WIFI_ERR_FAILED;
                 // A rejection seen on the RX path only marks the state; the
                 // firmware contexts are still up and have to come back down.
-                IwxConnectAbort();
+                ConnAbort();
                 return rc;
             }
             if (state == IwxConnStateId::Idle) return WIFI_ERR_FAILED;
-            if (g_iwx.State == IwxFwState::Error) {
-                IwxConnectAbort();
+            if (FwFailed()) {
+                ConnAbort();
                 return WIFI_ERR_FAILED;
             }
 
@@ -721,7 +893,7 @@ namespace Drivers::Net::Wifi {
             Sched::Schedule();
         }
 
-        IwxConnectAbort();
+        ConnAbort();
         return WIFI_ERR_TIMEOUT;
     }
 
@@ -732,7 +904,7 @@ namespace Drivers::Net::Wifi {
     // scan table is worth a (blocking) sweep to look for it.
     static int StartJoin(const char* ssid, const char* password, bool rescan) {
         if (!g_initialized || !ssid) return WIFI_ERR_NO_ADAPTER;
-        if (g_iwx.State != IwxFwState::Running) return WIFI_ERR_NO_ADAPTER;
+        if (!FwRunning()) return WIFI_ERR_NO_ADAPTER;
 
         // Locate the network in the most recent scan results: the firmware
         // contexts need its BSSID and channel, and the connect path needs its
@@ -809,17 +981,24 @@ namespace Drivers::Net::Wifi {
             if (!havePassword) return WIFI_ERR_NEED_KEY;
         }
 
-        if (!IwxConnectStart(bssid, channel, is5, ssid,
+        bool started = IsBrcmf()
+            ? BrcmfConnectStart(bssid, channel, ssid,
+                                havePassword ? password : nullptr,
+                                security == WIFI_SEC_OPEN ? nullptr : rsnIe,
+                                security == WIFI_SEC_OPEN ? 0 : rsnIeLen)
+            : IwxConnectStart(bssid, channel, is5, ssid,
                               havePassword ? password : nullptr,
                               security == WIFI_SEC_OPEN ? nullptr : rsnIe,
                               security == WIFI_SEC_OPEN ? 0 : rsnIeLen,
-                             beaconInterval, dtimPeriod)) {
-            // IwxConnectStart already logged the specific reason.  Only report
+                              beaconInterval, dtimPeriod);
+        if (!started) {
+            // The backend already logged the specific reason.  Only report
             // a security problem when that is genuinely what happened; a radio
             // or firmware failure needs completely different advice.
-            bool security_ = IwxConnectRefusedForSecurity();
-            auto state = (IwxConnStateId)IwxConnectState();
-            if (state != IwxConnStateId::Idle) IwxConnectAbort();
+            bool security_ = IsBrcmf() ? BrcmfConnectRefusedForSecurity()
+                                       : IwxConnectRefusedForSecurity();
+            auto state = (IwxConnStateId)ConnState();
+            if (state != IwxConnStateId::Idle) ConnAbort();
             return security_ ? WIFI_ERR_UNSUPPORTED : WIFI_ERR_FAILED;
         }
 
@@ -835,8 +1014,8 @@ namespace Drivers::Net::Wifi {
     int ConnectAsync(const char* ssid, const char* password) {
         // A join already in flight owns the firmware contexts; tear it down
         // rather than stacking a second one on top.
-        if (g_asyncConnect || (IwxConnectState() != (int)IwxConnStateId::Idle))
-            IwxConnectAbort();
+        if (g_asyncConnect || (ConnState() != (int)IwxConnStateId::Idle))
+            ConnAbort();
 
         g_asyncConnect = false;
         g_lastError = 0;
@@ -858,7 +1037,7 @@ namespace Drivers::Net::Wifi {
         if (!g_initialized) return -1;
         g_asyncConnect = false;
         g_lastError = 0;
-        IwxConnectAbort();
+        ConnAbort();
         return 0;
     }
 
@@ -881,11 +1060,11 @@ namespace Drivers::Net::Wifi {
         uint64_t now = Timekeeping::GetMilliseconds();
 
         if (g_scanDeadline) {
-            if (!g_iwx.ScanActive) {
+            if (!ScanActive()) {
                 g_scanDeadline = 0;
                 g_scanGeneration++;
             } else if (now >= g_scanDeadline) {
-                IwxAbortScan();
+                AbortScan();
                 g_scanDeadline = 0;
                 g_scanGeneration++;
             }
@@ -893,31 +1072,29 @@ namespace Drivers::Net::Wifi {
 
         if (!g_asyncConnect) return;
 
-        auto state = (IwxConnStateId)IwxConnectState();
+        auto state = (IwxConnStateId)ConnState();
         if (state == IwxConnStateId::Connected) {
             g_asyncConnect = false;
             g_lastError = 0;
             return;
         }
         if (state == IwxConnStateId::Failed) {
-            // Same reading as the blocking path: a handshake that exchanged
-            // EAPOL frames and then failed is almost always a wrong passphrase.
-            g_lastError = WpaGetState() == WpaState::Failed
-                          ? WIFI_ERR_AUTH : WIFI_ERR_FAILED;
+            // Same reading as the blocking path.
+            g_lastError = JoinFailedOnAuth() ? WIFI_ERR_AUTH : WIFI_ERR_FAILED;
             g_asyncConnect = false;
-            IwxConnectAbort();
+            ConnAbort();
             return;
         }
-        if (state == IwxConnStateId::Idle || g_iwx.State == IwxFwState::Error) {
+        if (state == IwxConnStateId::Idle || FwFailed()) {
             g_lastError = WIFI_ERR_FAILED;
             g_asyncConnect = false;
-            if (state != IwxConnStateId::Idle) IwxConnectAbort();
+            if (state != IwxConnStateId::Idle) ConnAbort();
             return;
         }
         if (now >= g_asyncConnectDeadline) {
             g_lastError = WIFI_ERR_TIMEOUT;
             g_asyncConnect = false;
-            IwxConnectAbort();
+            ConnAbort();
         }
     }
 
@@ -927,13 +1104,33 @@ namespace Drivers::Net::Wifi {
 
     static RxCallback g_rxCallback = nullptr;
 
-    const uint8_t* GetMacAddress() { return g_iwx.Nvm.HwAddr; }
+    const uint8_t* GetMacAddress() { return IsBrcmf() ? g_brcmf.Mac : g_iwx.Nvm.HwAddr; }
 
-    bool IsLinkUp() { return g_initialized && IwxLinkUp(); }
+    bool IsLinkUp() { return g_initialized && LinkUp(); }
 
     bool SendPacket(const uint8_t* data, uint16_t length) {
         if (!IsLinkUp()) return false;
-        return IwxConnectSendEthernet(data, length);
+        return IsBrcmf() ? BrcmfSendEthernet(data, length)
+                         : IwxConnectSendEthernet(data, length);
+    }
+
+    // =========================================================================
+    // Supplicant hooks (Wpa.hpp): to whichever backend is running the join
+    // =========================================================================
+
+    bool WpaTxEapol(const uint8_t* body, uint32_t len) {
+        return IsBrcmf() ? BrcmfWpaTxEapol(body, len) : IwxWpaTxEapol(body, len);
+    }
+
+    bool WpaInstallPtk(const uint8_t* tk, uint32_t tkLen, uint8_t cipher) {
+        return IsBrcmf() ? BrcmfWpaInstallPtk(tk, tkLen, cipher)
+                         : IwxWpaInstallPtk(tk, tkLen, cipher);
+    }
+
+    bool WpaInstallGtk(const uint8_t* gtk, uint32_t gtkLen, uint8_t keyIdx,
+                       uint8_t cipher, const uint8_t* rsc) {
+        return IsBrcmf() ? BrcmfWpaInstallGtk(gtk, gtkLen, keyIdx, cipher, rsc)
+                         : IwxWpaInstallGtk(gtk, gtkLen, keyIdx, cipher, rsc);
     }
 
     void SetRxCallback(RxCallback callback) { g_rxCallback = callback; }

kernel/src/Drivers/Net/Wifi/Wifi.hpp

@@ -14,6 +14,11 @@ namespace Drivers::Net::Wifi {
     // PCI driver-table probe entry point.
     bool Probe(const Pci::PciDevice& dev);
 
+    // The Raspberry Pi 4/400's Broadcom/Cypress radio, on the SDIO
+    // controller (Arasan) at `mmioPhys`. Claims it; the bring-up waits for
+    // the root filesystem like the PCI path.
+    bool ProbeBroadcomSdio(uint64_t mmioPhys, const char* name);
+
     // Complete the firmware-dependent bring-up once the ramdisk is mounted.
     // Cheap no-op unless a device is waiting.  Called from the idle loop.
     void ServiceDeferredInit();
@@ -25,6 +30,11 @@ namespace Drivers::Net::Wifi {
     // deadline, or a due recovery attempt needs idle-context servicing.
     bool HasDeferredWork();
 
+    // A polled adapter (the Broadcom radio has no interrupt in use) is due a
+    // sweep. Cheap and safe from the timer interrupt: the tick uses it to
+    // force a bottom-half pass when the cores never go idle.
+    bool HasPolledWork();
+
     bool IsInitialized();
     bool IsPresent();
 

kernel/src/Drivers/Net/Wifi/Wpa.hpp

@@ -66,7 +66,8 @@ namespace Drivers::Net::Wifi {
     bool WpaParseApRsn(const uint8_t* ie, uint32_t len, WpaConfig& cfg);
 
     // -------------------------------------------------------------------------
-    // Hooks implemented by the MLME (IwxConnect.cpp)
+    // Hooks into the driver running the join (Wifi.cpp routes them to
+    // IwxConnect.cpp or BrcmfConnect.cpp)
     // -------------------------------------------------------------------------
 
     // Transmit an EAPOL frame body (802.1X header included) to the AP.

kernel/src/Drivers/Platform/RpiFirmware.cpp

@@ -0,0 +1,284 @@
+/*
+    * RpiFirmware.cpp
+    * Raspberry Pi VideoCore firmware mailbox (property interface)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "RpiFirmware.hpp"
+#include <ACPI/AcpiDevices.hpp>
+#include <Arch/Cpu.hpp>
+#include <CppLib/Spinlock.hpp>
+#include <CppLib/Stream.hpp>
+#include <Libraries/Memory.hpp>
+#include <Boot/Boot.hpp>
+#include <Memory/Memmap.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// Property interface (raspberrypi/firmware wiki, "Mailbox property
+// interface"): the ARM writes a message buffer's bus address, tagged with
+// channel 8, to mailbox 1 and reads the same address back from mailbox 0
+// once the firmware has written its responses into the buffer.
+//
+//   u32 size, u32 code (0 = request; 0x80000000 = success),
+//   tags: { u32 id, u32 value buffer size, u32 req/resp (bit 31 + length),
+//           value buffer ... }, u32 0 (end tag)
+
+using namespace Kt;
+
+namespace Drivers::Platform::RpiFirmware {
+
+    // Mailbox registers (offsets from the block ACPI describes, 0x7E00B880)
+    static constexpr uint32_t MBOX_READ    = 0x00;   // mailbox 0: VC -> ARM
+    static constexpr uint32_t MBOX0_STATUS = 0x18;
+    static constexpr uint32_t MBOX_WRITE   = 0x20;   // mailbox 1: ARM -> VC
+    static constexpr uint32_t MBOX1_STATUS = 0x38;
+    static constexpr uint32_t STATUS_FULL  = 1u << 31;
+    static constexpr uint32_t STATUS_EMPTY = 1u << 30;
+
+    static constexpr uint32_t CHANNEL_PROPERTY = 8;
+
+    static constexpr uint32_t CODE_REQUEST  = 0x00000000;
+    static constexpr uint32_t CODE_SUCCESS  = 0x80000000;
+    static constexpr uint32_t TAG_RESPONSE  = 0x80000000;
+
+    static constexpr uint32_t TAG_GET_BOARD_MAC   = 0x00010003;
+    static constexpr uint32_t TAG_SET_CLOCK_STATE = 0x00038001;
+    static constexpr uint32_t TAG_GET_CLOCK_RATE  = 0x00030002;
+    static constexpr uint32_t TAG_GET_GPIO_STATE  = 0x00030041;
+    static constexpr uint32_t TAG_SET_GPIO_STATE  = 0x00038041;
+    static constexpr uint32_t TAG_GET_GPIO_CONFIG = 0x00030043;
+    static constexpr uint32_t TAG_SET_GPIO_CONFIG = 0x00038043;
+
+    // The VideoCore sees the first GiB of SDRAM at bus address 0xC0000000
+    // (the uncached alias), and nothing above it, so the message buffer
+    // must sit below 1 GiB.
+    static constexpr uint64_t VC_BUS_ALIAS = 0xC0000000;
+    static constexpr uint64_t VC_DMA_LIMIT = 0x40000000;
+
+    static constexpr uint64_t TimeoutMs = 200;
+
+    static volatile uint8_t* g_mmio = nullptr;
+    static volatile uint32_t* g_buffer = nullptr;   // uncached, below 1 GiB
+    static uint64_t g_bufferPhys = 0;
+    static bool g_probed = false;
+    static kcp::Mutex g_lock;
+
+    static inline uint32_t Read32(uint32_t offset) {
+        return *(volatile uint32_t*)(g_mmio + offset);
+    }
+
+    static inline void Write32(uint32_t offset, uint32_t value) {
+        *(volatile uint32_t*)(g_mmio + offset) = value;
+    }
+
+    // A page the VideoCore can reach. The page-frame allocator only manages
+    // the largest run of usable RAM, which on a Pi 4 with the 3 GB limit is
+    // 0x40000000 and up, so it never has one. The usable regions below 1 GiB
+    // are otherwise untouched: take the last page of the highest one.
+    static void* AllocateLowPage(uint64_t& phys) {
+        const auto& map = montauk::boot::Info().memoryMap;
+        Memory::LargestSection pool = Memory::Scan(map);
+
+        uint64_t best = 0;
+        for (uint64_t i = 0; i < map.count; i++) {
+            const auto& r = map.regions[i];
+            if (r.kind != montauk::boot::MemoryKind::Usable || r.base == 0) continue;
+            if (r.base == pool.address) continue;            // the allocator's
+            uint64_t end = r.base + r.length;
+            if (end > VC_DMA_LIMIT) end = VC_DMA_LIMIT;
+            if (end < r.base + 0x1000) continue;
+            uint64_t page = (end - 0x1000) & ~0xFFFULL;
+            if (page >= r.base && page > best) best = page;
+        }
+        if (best == 0) return nullptr;
+
+        // Zero it through its cacheable HHDM alias, push the lines out, then
+        // make the alias uncached so neither side needs cache maintenance.
+        void* virt = (void*)Memory::HHDM(best);
+        memset(virt, 0, 0x1000);
+        Arch::InvalidateDataCache(virt, 0x1000);
+        Memory::VMM::g_paging->MapWC(best, (uint64_t)virt);
+        phys = best;
+        return virt;
+    }
+
+    bool Initialize() {
+        if (g_probed) return g_buffer != nullptr;
+        g_probed = true;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BCM2849", info)) return false;
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            KernelLogStream(WARNING, "RpiFw") << "Mailbox " << info.Path << ": no register block in _CRS";
+            return false;
+        }
+
+        uint64_t page = base & ~0xFFFULL;
+        Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
+        if (((base + 0x40) & ~0xFFFULL) != page) {
+            Memory::VMM::g_paging->MapMMIO(page + 0x1000, Memory::HHDM(page + 0x1000));
+        }
+        g_mmio = (volatile uint8_t*)Memory::HHDM(base);
+
+        uint64_t phys = 0;
+        void* buffer = AllocateLowPage(phys);
+        if (buffer == nullptr) {
+            KernelLogStream(ERROR, "RpiFw") << "No message buffer below 1 GiB";
+            g_mmio = nullptr;
+            return false;
+        }
+        g_buffer = (volatile uint32_t*)buffer;
+        g_bufferPhys = phys;
+
+        KernelLogStream(OK, "RpiFw") << "VideoCore mailbox at 0x" << base::hex << base
+            << ", buffer 0x" << phys << base::dec;
+        return true;
+    }
+
+    bool IsPresent() {
+        return Initialize();
+    }
+
+    // Send the message in g_buffer and wait for the firmware's reply.
+    // Caller holds g_lock.
+    static bool Call() {
+        uint32_t message = (uint32_t)((g_bufferPhys | VC_BUS_ALIAS) & ~0xFULL) | CHANNEL_PROPERTY;
+
+        // Drop any stale replies (a previous call that timed out).
+        for (int i = 0; i < 16 && !(Read32(MBOX0_STATUS) & STATUS_EMPTY); i++) {
+            (void)Read32(MBOX_READ);
+        }
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Read32(MBOX1_STATUS) & STATUS_FULL) {
+            if (Timekeeping::GetMilliseconds() - start > TimeoutMs) return false;
+            Arch::Pause();
+        }
+
+        // The buffer is uncached; the fence orders its stores before the
+        // doorbell write the firmware reacts to.
+        Arch::MemoryFence();
+        Write32(MBOX_WRITE, message);
+
+        start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            while (Read32(MBOX0_STATUS) & STATUS_EMPTY) {
+                if (Timekeeping::GetMilliseconds() - start > TimeoutMs) {
+                    KernelLogStream(WARNING, "RpiFw") << "Mailbox call timed out";
+                    return false;
+                }
+                Arch::Pause();
+            }
+            uint32_t reply = Read32(MBOX_READ);
+            if (reply == message) break;
+        }
+        Arch::MemoryFence();
+        return g_buffer[1] == CODE_SUCCESS;
+    }
+
+    // One-tag property call. `values` holds valueWords request words and
+    // receives up to valueWords response words. Returns the response length
+    // in bytes, or -1 on failure.
+    static int PropertyCall(uint32_t tag, uint32_t* values, uint32_t valueWords) {
+        if (!Initialize()) return -1;
+        if (valueWords > 16) return -1;
+
+        g_lock.Acquire();
+        uint32_t words = 0;
+        g_buffer[words++] = 0;                    // size, filled in below
+        g_buffer[words++] = CODE_REQUEST;
+        g_buffer[words++] = tag;
+        g_buffer[words++] = valueWords * 4;       // value buffer size
+        g_buffer[words++] = 0;                    // request
+        uint32_t valueStart = words;
+        for (uint32_t i = 0; i < valueWords; i++) g_buffer[words++] = values[i];
+        g_buffer[words++] = 0;                    // end tag
+        g_buffer[0] = words * 4;
+
+        // The firmware does not set the response bit for every tag: the
+        // expander GPIO tags answer through value[0] alone and leave the
+        // status word 0. Linux checks only the message code, so do the same
+        // and take the whole value buffer back when the bit is missing; each
+        // caller validates the values it gets.
+        int result = -1;
+        bool answered = Call();
+        uint32_t status = g_buffer[valueStart - 1];
+        if (answered) {
+            uint32_t length = (status & TAG_RESPONSE) ? status & ~TAG_RESPONSE : valueWords * 4;
+            uint32_t copy = (length + 3) / 4;
+            if (copy > valueWords) copy = valueWords;
+            for (uint32_t i = 0; i < copy; i++) values[i] = g_buffer[valueStart + i];
+            result = (int)length;
+        }
+        uint32_t code = g_buffer[1];
+        g_lock.Release();
+
+        if (result < 0) {
+            KernelLogStream(WARNING, "RpiFw") << "Property tag 0x" << base::hex << (uint64_t)tag
+                << " failed: message code 0x" << (uint64_t)code << base::dec;
+        }
+        return result;
+    }
+
+    bool GetClockRate(uint32_t clockId, uint32_t& hz) {
+        uint32_t v[2] = { clockId, 0 };
+        if (PropertyCall(TAG_GET_CLOCK_RATE, v, 2) < 8 || v[0] != clockId) return false;
+        hz = v[1];
+        return hz != 0;
+    }
+
+    // State bit 0 = on; the answer echoes the clock ID and its new state
+    // (bit 1 set: no such clock).
+    bool SetClockState(uint32_t clockId, bool on) {
+        uint32_t v[2] = { clockId, on ? 1u : 0u };
+        if (PropertyCall(TAG_SET_CLOCK_STATE, v, 2) < 8 || v[0] != clockId) return false;
+        return (v[1] & 2) == 0 && ((v[1] & 1) != 0) == on;
+    }
+
+    // The GPIO tags answer in place of the GPIO number with a status word,
+    // 0 for success (as Linux's raspberrypi-exp-gpio driver checks).
+    bool SetGpioState(uint32_t gpio, bool high) {
+        uint32_t v[2] = { gpio, high ? 1u : 0u };
+        return PropertyCall(TAG_SET_GPIO_STATE, v, 2) >= 4 && v[0] == 0;
+    }
+
+    // As Linux's rpi_exp_gpio_dir_out: keep the line's polarity, make it an
+    // output driving `high`. A line the firmware still has as an input
+    // refuses SET_GPIO_STATE.
+    bool SetGpioOutput(uint32_t gpio, bool high) {
+        uint32_t polarity = 0;
+        uint32_t cfg[5] = { gpio, 0, 0, 0, 0 };   // gpio, direction, polarity, term_en, term_pull_up
+        if (PropertyCall(TAG_GET_GPIO_CONFIG, cfg, 5) >= 12 && cfg[0] == 0) polarity = cfg[2];
+
+        uint32_t v[6] = { gpio, 1 /* output */, polarity, 0, 0, high ? 1u : 0u };
+        if (PropertyCall(TAG_SET_GPIO_CONFIG, v, 6) < 4 || v[0] != 0) return false;
+        return true;
+    }
+
+    bool GetGpioState(uint32_t gpio, bool& high) {
+        uint32_t v[2] = { gpio, 0 };
+        if (PropertyCall(TAG_GET_GPIO_STATE, v, 2) < 8 || v[0] != 0) return false;
+        high = v[1] != 0;
+        return true;
+    }
+
+    bool GetBoardMacAddress(uint8_t mac[6]) {
+        uint32_t v[2] = {};
+        if (PropertyCall(TAG_GET_BOARD_MAC, v, 2) < 6) return false;
+        const uint8_t* bytes = (const uint8_t*)v;
+        bool allZero = true;
+        for (int i = 0; i < 6; i++) {
+            mac[i] = bytes[i];
+            if (bytes[i] != 0) allZero = false;
+        }
+        return !allZero;
+    }
+
+}

kernel/src/Drivers/Platform/RpiFirmware.hpp

@@ -0,0 +1,53 @@
+/*
+    * RpiFirmware.hpp
+    * Raspberry Pi VideoCore firmware mailbox (property interface)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// The VideoCore firmware owns parts of the board the ARM cores cannot reach
+// directly: the GPIO expander that powers the Wi-Fi/Bluetooth radio, the
+// clock tree, the board's OTP (serial number, MAC address). The ARM side asks
+// for them through the property channel of the VideoCore mailbox, which
+// ACPI describes as BCM2849 (\_SB.GDV0.RPIQ) on the Pi Firmware Task Force
+// UEFI firmware.
+//
+// All calls are synchronous, take a process-context lock, and must not be
+// made from interrupt context. They return false when there is no mailbox
+// (any machine but a Pi) or the firmware rejects the request.
+
+namespace Drivers::Platform::RpiFirmware {
+
+    // Find the mailbox through ACPI and allocate its message buffer.
+    // Idempotent; the other calls initialize on first use.
+    bool Initialize();
+
+    bool IsPresent();
+
+    // Clock identifiers (the firmware's numbering).
+    constexpr uint32_t CLOCK_EMMC = 1;   // legacy EMMC (Arasan), runs the Wi-Fi SDIO bus
+    constexpr uint32_t CLOCK_CORE = 4;   // VPU core clock, feeds the mini UART
+
+    // Current rate of a firmware clock, in Hz.
+    bool GetClockRate(uint32_t clockId, uint32_t& hz);
+
+    // Gate a firmware clock on or off.
+    bool SetClockState(uint32_t clockId, bool on);
+
+    // The firmware's GPIO expander lines are numbered from 128.
+    constexpr uint32_t EXPGPIO_BASE  = 128;
+    constexpr uint32_t EXPGPIO_BT_ON = EXPGPIO_BASE + 0;   // Bluetooth power (high = on)
+    constexpr uint32_t EXPGPIO_WL_ON = EXPGPIO_BASE + 1;   // Wi-Fi power (high = on)
+
+    bool SetGpioState(uint32_t gpio, bool high);
+    // Configure the line as an output, then drive it (SET_GPIO_CONFIG).
+    bool SetGpioOutput(uint32_t gpio, bool high);
+    bool GetGpioState(uint32_t gpio, bool& high);
+
+    // The board's Ethernet MAC address from OTP, in transmission order
+    // (mac[0] is the first octet of the OUI).
+    bool GetBoardMacAddress(uint8_t mac[6]);
+
+}

kernel/src/Drivers/Platform/RpiGpio.cpp

@@ -0,0 +1,69 @@
+/*
+    * RpiGpio.cpp
+    * BCM2711 GPIO pin function and pull control
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "RpiGpio.hpp"
+#include <ACPI/AcpiDevices.hpp>
+#include <CppLib/Spinlock.hpp>
+#include <CppLib/Stream.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Terminal/Terminal.hpp>
+
+using namespace Kt;
+
+namespace Drivers::Platform::RpiGpio {
+
+    // BCM2711 ARM Peripherals, chapter 5
+    static constexpr uint32_t REG_GPFSEL0         = 0x00;   // 10 pins per register, 3 bits each
+    static constexpr uint32_t REG_PUP_PDN_CNTRL0  = 0xE4;   // 16 pins per register, 2 bits each
+    static constexpr uint32_t PinCount            = 58;
+
+    static volatile uint8_t* g_mmio = nullptr;
+    static bool g_probed = false;
+    static kcp::Spinlock g_lock;
+
+    static bool Initialize() {
+        if (g_probed) return g_mmio != nullptr;
+        g_probed = true;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BCM2845", info)) return false;
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            return false;
+        }
+
+        uint64_t page = base & ~0xFFFULL;
+        Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
+        g_mmio = (volatile uint8_t*)Memory::HHDM(base);
+        return true;
+    }
+
+    static void Modify(uint32_t offset, uint32_t mask, uint32_t value) {
+        g_lock.Acquire();
+        volatile uint32_t* reg = (volatile uint32_t*)(g_mmio + offset);
+        *reg = (*reg & ~mask) | (value & mask);
+        g_lock.Release();
+    }
+
+    bool SetFunction(uint32_t pin, Function function) {
+        if (pin >= PinCount || !Initialize()) return false;
+        uint32_t shift = (pin % 10) * 3;
+        Modify(REG_GPFSEL0 + (pin / 10) * 4, 7u << shift, (uint32_t)function << shift);
+        return true;
+    }
+
+    bool SetPull(uint32_t pin, Pull pull) {
+        if (pin >= PinCount || !Initialize()) return false;
+        uint32_t shift = (pin % 16) * 2;
+        Modify(REG_PUP_PDN_CNTRL0 + (pin / 16) * 4, 3u << shift, (uint32_t)pull << shift);
+        return true;
+    }
+
+}

kernel/src/Drivers/Platform/RpiGpio.hpp

@@ -0,0 +1,37 @@
+/*
+    * RpiGpio.hpp
+    * BCM2711 GPIO pin function and pull control
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// The SoC's own GPIO block (ACPI BCM2845, \_SB.GDV0.GPI0), used to route
+// pins to the peripheral that drives them. On Linux the device tree's
+// pinctrl does this; booted through ACPI, nothing does it for us. Only the
+// BCM2711 (Raspberry Pi 4/400) pull-control layout is implemented.
+
+namespace Drivers::Platform::RpiGpio {
+
+    enum class Function : uint8_t {
+        Input  = 0b000,
+        Output = 0b001,
+        Alt0   = 0b100,
+        Alt1   = 0b101,
+        Alt2   = 0b110,
+        Alt3   = 0b111,
+        Alt4   = 0b011,
+        Alt5   = 0b010,
+    };
+
+    enum class Pull : uint8_t {
+        None = 0b00,
+        Up   = 0b01,
+        Down = 0b10,
+    };
+
+    bool SetFunction(uint32_t pin, Function function);
+    bool SetPull(uint32_t pin, Pull pull);
+
+}

kernel/src/Drivers/Serial/BtUart.cpp

@@ -0,0 +1,488 @@
+/*
+    * BtUart.cpp
+    * Interrupt-driven UART to an on-board Bluetooth controller
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "BtUart.hpp"
+#include <ACPI/GsiRouting.hpp>
+#include <Arch/Cpu.hpp>
+#include <Arch/Interrupts.hpp>
+#include <CppLib/Spinlock.hpp>
+#include <CppLib/Stream.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Sched/Scheduler.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+#include <atomic>
+
+using namespace Kt;
+
+namespace Drivers::Serial::BtUart {
+
+    // =========================================================================
+    // Registers
+    // =========================================================================
+
+    // Mini UART, offsets from the AUX block (BCM2711 ARM Peripherals 2.2,
+    // with the elinux.org BCM2835 datasheet errata: IER bit 0 enables the
+    // receive interrupt and bit 1 the transmit one, IER bits 3:2 must be set
+    // for interrupts to be raised, LCR needs 0b11 for 8 data bits).
+    static constexpr uint32_t AUX_ENABLES   = 0x04;
+    static constexpr uint32_t MU_IO         = 0x40;
+    static constexpr uint32_t MU_IER        = 0x44;
+    static constexpr uint32_t MU_IIR        = 0x48;
+    static constexpr uint32_t MU_LCR        = 0x4C;
+    static constexpr uint32_t MU_MCR        = 0x50;
+    static constexpr uint32_t MU_LSR        = 0x54;
+    static constexpr uint32_t MU_CNTL       = 0x60;
+    static constexpr uint32_t MU_BAUD       = 0x68;
+
+    static constexpr uint32_t AUX_ENABLE_MU     = 1u << 0;
+    static constexpr uint32_t MU_IER_RX         = 1u << 0;
+    static constexpr uint32_t MU_IER_TX         = 1u << 1;
+    static constexpr uint32_t MU_IER_REQUIRED   = 3u << 2;
+    static constexpr uint32_t MU_IIR_FLUSH      = 0xC6;      // clear both FIFOs, FIFOs on
+    static constexpr uint32_t MU_LCR_8BIT       = 0x03;
+    static constexpr uint32_t MU_MCR_RTS        = 1u << 1;   // RTS pin low (asserted)
+    static constexpr uint32_t MU_LSR_DATA_READY = 1u << 0;
+    static constexpr uint32_t MU_LSR_OVERRUN    = 1u << 1;
+    static constexpr uint32_t MU_LSR_TX_SPACE   = 1u << 5;   // FIFO accepts a byte
+    static constexpr uint32_t MU_LSR_TX_IDLE    = 1u << 6;   // FIFO empty, shifter idle
+    static constexpr uint32_t MU_CNTL_RXEN      = 1u << 0;
+    static constexpr uint32_t MU_CNTL_TXEN      = 1u << 1;
+    // Auto RTS at the default level (deassert with 3 FIFO spaces left).
+    // Bit 6 picks the level auto RTS asserts: set = low, clear (the reset
+    // value) = high. A UART's RTS is active low, so it must be set: left
+    // clear, RTS reads "stop" to the controller the moment auto RTS is on.
+    // Bits 3 and 7 are the same for CTS: on a Pi 400, with bit 7 clear the
+    // transmitter held its bytes while the controller drove CTS low (ready).
+    static constexpr uint32_t MU_CNTL_AUTORTS   = 1u << 2;
+    static constexpr uint32_t MU_CNTL_AUTOCTS   = 1u << 3;
+    static constexpr uint32_t MU_CNTL_RTS_LOW   = 1u << 6;
+    static constexpr uint32_t MU_CNTL_CTS_LOW   = 1u << 7;
+
+    // PL011 (ARM PrimeCell UART TRM)
+    static constexpr uint32_t PL_DR   = 0x00;
+    static constexpr uint32_t PL_FR   = 0x18;
+    static constexpr uint32_t PL_IBRD = 0x24;
+    static constexpr uint32_t PL_FBRD = 0x28;
+    static constexpr uint32_t PL_LCRH = 0x2C;
+    static constexpr uint32_t PL_CR   = 0x30;
+    static constexpr uint32_t PL_IFLS = 0x34;
+    static constexpr uint32_t PL_IMSC = 0x38;
+    static constexpr uint32_t PL_ICR  = 0x44;
+
+    static constexpr uint32_t PL_DR_OE    = 1u << 11;
+    static constexpr uint32_t PL_FR_BUSY  = 1u << 3;
+    static constexpr uint32_t PL_FR_RXFE  = 1u << 4;
+    static constexpr uint32_t PL_FR_TXFF  = 1u << 5;
+    static constexpr uint32_t PL_FR_TXFE  = 1u << 7;
+    static constexpr uint32_t PL_LCRH_FEN  = 1u << 4;
+    static constexpr uint32_t PL_LCRH_8BIT = 3u << 5;
+    static constexpr uint32_t PL_CR_UARTEN = 1u << 0;
+    static constexpr uint32_t PL_CR_TXE    = 1u << 8;
+    static constexpr uint32_t PL_CR_RXE    = 1u << 9;
+    static constexpr uint32_t PL_CR_RTS    = 1u << 11;
+    static constexpr uint32_t PL_CR_RTSEN  = 1u << 14;
+    static constexpr uint32_t PL_CR_CTSEN  = 1u << 15;
+    static constexpr uint32_t PL_INT_RX    = 1u << 4;
+    static constexpr uint32_t PL_INT_TX    = 1u << 5;
+    static constexpr uint32_t PL_INT_RT    = 1u << 6;    // receive timeout
+    static constexpr uint32_t PL_INT_OE    = 1u << 10;
+    static constexpr uint32_t PL_IFLS_RX_HALF = 2u << 3;
+    static constexpr uint32_t PL_IFLS_TX_EIGHTH = 0;
+
+    // =========================================================================
+    // State
+    // =========================================================================
+
+    static constexpr uint32_t RX_RING_SIZE = 16384;   // powers of two
+    static constexpr uint32_t TX_RING_SIZE = 16384;
+
+    static Kind g_kind = Kind::MiniUart;
+    static volatile uint8_t* g_mmio = nullptr;
+    static uint32_t g_clockHz = 0;
+    static uint32_t g_baud = 0;
+    static bool g_flowControl = false;
+    static bool g_interrupt = false;
+    static bool g_initialized = false;
+
+    // Guards the hardware and the transmit ring: taken by the IRQ handler,
+    // Service() and every producer, with interrupts off.
+    static kcp::Spinlock g_hwLock;
+
+    // Receive ring: produced under g_hwLock, consumed by Read() (one reader).
+    static uint8_t g_rxRing[RX_RING_SIZE];
+    static std::atomic<uint32_t> g_rxHead{0};
+    static std::atomic<uint32_t> g_rxTail{0};
+
+    // Transmit ring: entirely under g_hwLock. g_txCount mirrors the fill for
+    // lock-free readers.
+    static uint8_t g_txRing[TX_RING_SIZE];
+    static uint32_t g_txHead = 0;
+    static uint32_t g_txTail = 0;
+    static std::atomic<uint32_t> g_txCount{0};
+    static bool g_txIrqEnabled = false;
+
+    static Stats g_stats = {};
+
+    static inline uint32_t Read32(uint32_t offset) {
+        return *(volatile uint32_t*)(g_mmio + offset);
+    }
+
+    static inline void Write32(uint32_t offset, uint32_t value) {
+        *(volatile uint32_t*)(g_mmio + offset) = value;
+    }
+
+    // =========================================================================
+    // FIFO service (caller holds g_hwLock)
+    // =========================================================================
+
+    static inline void PushRx(uint8_t byte) {
+        uint32_t head = g_rxHead.load(std::memory_order_relaxed);
+        if (head - g_rxTail.load(std::memory_order_acquire) >= RX_RING_SIZE) {
+            g_stats.rxDropped++;
+            return;
+        }
+        g_rxRing[head & (RX_RING_SIZE - 1)] = byte;
+        g_rxHead.store(head + 1, std::memory_order_release);
+        g_stats.rxBytes++;
+    }
+
+    static bool DrainRxLocked() {
+        bool any = false;
+        if (g_kind == Kind::MiniUart) {
+            for (;;) {
+                uint32_t lsr = Read32(MU_LSR);
+                if (lsr & MU_LSR_OVERRUN) g_stats.overruns++;
+                if (!(lsr & MU_LSR_DATA_READY)) break;
+                PushRx((uint8_t)Read32(MU_IO));
+                any = true;
+            }
+        } else {
+            while (!(Read32(PL_FR) & PL_FR_RXFE)) {
+                uint32_t data = Read32(PL_DR);
+                if (data & PL_DR_OE) g_stats.overruns++;
+                PushRx((uint8_t)data);
+                any = true;
+            }
+            Write32(PL_ICR, PL_INT_RX | PL_INT_RT | PL_INT_OE);
+        }
+        return any;
+    }
+
+    static void SetTxIrqLocked(bool on) {
+        if (on == g_txIrqEnabled || !g_interrupt) return;
+        g_txIrqEnabled = on;
+        if (g_kind == Kind::MiniUart) {
+            Write32(MU_IER, MU_IER_RX | MU_IER_REQUIRED | (on ? MU_IER_TX : 0));
+        } else {
+            Write32(PL_IMSC, PL_INT_RX | PL_INT_RT | PL_INT_OE | (on ? PL_INT_TX : 0));
+        }
+    }
+
+    // Move queued bytes into the transmit FIFO. The transmit interrupt stays
+    // enabled exactly while bytes remain queued: the PL011's only fires on
+    // crossing its FIFO threshold, so the FIFO must be refilled here first.
+    static void FillTxLocked() {
+        while (g_txHead != g_txTail) {
+            bool space = (g_kind == Kind::MiniUart)
+                ? (Read32(MU_LSR) & MU_LSR_TX_SPACE) != 0
+                : (Read32(PL_FR) & PL_FR_TXFF) == 0;
+            if (!space) break;
+            uint8_t byte = g_txRing[g_txTail & (TX_RING_SIZE - 1)];
+            g_txTail++;
+            Write32(g_kind == Kind::MiniUart ? MU_IO : PL_DR, byte);
+            g_stats.txBytes++;
+        }
+        g_txCount.store(g_txHead - g_txTail, std::memory_order_release);
+        if (g_kind == Kind::Pl011) Write32(PL_ICR, PL_INT_TX);
+        SetTxIrqLocked(g_txHead != g_txTail);
+    }
+
+    static bool TransmitterIdleLocked() {
+        if (g_txHead != g_txTail) return false;
+        if (g_kind == Kind::MiniUart) return (Read32(MU_LSR) & MU_LSR_TX_IDLE) != 0;
+        uint32_t fr = Read32(PL_FR);
+        return (fr & PL_FR_TXFE) && !(fr & PL_FR_BUSY);
+    }
+
+    static void HandleInterrupt(uint8_t, bool fromUser) {
+        g_hwLock.Acquire();
+        g_stats.interrupts++;
+        bool received = DrainRxLocked();
+        FillTxLocked();
+        g_hwLock.Release();
+
+        // Received bytes are parsed in process context. When this interrupt
+        // landed in user mode, ask for an idle-context pass at once (as the
+        // timer does for USB) instead of waiting for the core to go idle.
+        if (received && fromUser) Sched::RequestKernelWork();
+    }
+
+    // =========================================================================
+    // Baud rate (caller holds g_hwLock, transmitter idle)
+    // =========================================================================
+
+    static bool WithinTolerance(uint64_t actual, uint32_t wanted) {
+        uint64_t diff = actual > wanted ? actual - wanted : wanted - actual;
+        return diff * 40 <= wanted;   // 2.5%
+    }
+
+    // The divisor for `baud`: mini UART, baud = clock / (8 * divisor) (the
+    // register holds divisor - 1); PL011, clock / (16 * baud) in 16.6 fixed
+    // point (IBRD = divisor >> 6, FBRD = divisor & 63).
+    static bool ComputeDivisor(uint32_t baud, uint64_t& divisor) {
+        if (baud == 0 || g_clockHz == 0) return false;
+        if (g_kind == Kind::MiniUart) {
+            divisor = ((uint64_t)g_clockHz + 4ull * baud) / (8ull * baud);
+            if (divisor < 1 || divisor > 0x10000) return false;
+            return WithinTolerance((uint64_t)g_clockHz / (8 * divisor), baud);
+        }
+        divisor = ((uint64_t)g_clockHz * 8 / baud + 1) / 2;
+        if ((divisor >> 6) < 1 || (divisor >> 6) > 0xFFFF) return false;
+        return WithinTolerance((uint64_t)g_clockHz * 4 / divisor, baud);
+    }
+
+    static bool ProgramBaudLocked(uint32_t baud) {
+        uint64_t divisor = 0;
+        if (!ComputeDivisor(baud, divisor)) return false;
+
+        if (g_kind == Kind::MiniUart) {
+            uint32_t cntl = Read32(MU_CNTL);
+            Write32(MU_CNTL, 0);
+            Write32(MU_BAUD, (uint32_t)(divisor - 1));
+            Write32(MU_CNTL, cntl);
+        } else {
+            uint32_t ibrd = (uint32_t)(divisor >> 6);
+            uint32_t fbrd = (uint32_t)(divisor & 63);
+            uint32_t cr = Read32(PL_CR);
+            Write32(PL_CR, 0);
+            for (int i = 0; i < 100000 && (Read32(PL_FR) & PL_FR_BUSY); i++) Arch::Pause();
+            Write32(PL_LCRH, 0);                     // flush the FIFOs
+            Write32(PL_IBRD, ibrd);
+            Write32(PL_FBRD, fbrd);
+            Write32(PL_LCRH, PL_LCRH_8BIT | PL_LCRH_FEN);   // latches IBRD/FBRD
+            Write32(PL_CR, cr);
+        }
+        g_baud = baud;
+        return true;
+    }
+
+    // =========================================================================
+    // Public API
+    // =========================================================================
+
+    bool Initialize(const Config& config) {
+        if (g_initialized) return true;
+        if (config.mmioPhys == 0 || config.clockHz == 0) return false;
+
+        g_kind = config.kind;
+        g_clockHz = config.clockHz;
+
+        // The mini UART lives 0x40 into the AUX block; accept either base.
+        uint64_t base = config.mmioPhys;
+        if (g_kind == Kind::MiniUart && (base & 0xFF) == MU_IO) base -= MU_IO;
+        uint64_t page = base & ~0xFFFULL;
+        Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
+        g_mmio = (volatile uint8_t*)Memory::HHDM(base);
+
+        g_hwLock.Acquire();
+        if (g_kind == Kind::MiniUart) {
+            // AUX_ENABLES also gates SPI1/SPI2; leave their bits alone.
+            Write32(AUX_ENABLES, Read32(AUX_ENABLES) | AUX_ENABLE_MU);
+            Write32(MU_CNTL, 0);
+            Write32(MU_IER, 0);
+            Write32(MU_LCR, MU_LCR_8BIT);
+            Write32(MU_MCR, MU_MCR_RTS);
+            Write32(MU_IIR, MU_IIR_FLUSH);
+            Write32(MU_CNTL, MU_CNTL_RXEN | MU_CNTL_TXEN);
+        } else {
+            Write32(PL_CR, 0);
+            Write32(PL_IMSC, 0);
+            Write32(PL_ICR, 0x7FF);
+            Write32(PL_IFLS, PL_IFLS_RX_HALF | PL_IFLS_TX_EIGHTH);
+            Write32(PL_LCRH, PL_LCRH_8BIT | PL_LCRH_FEN);
+            Write32(PL_CR, PL_CR_UARTEN | PL_CR_TXE | PL_CR_RXE | PL_CR_RTS);
+        }
+        bool baudOk = ProgramBaudLocked(115200);
+        while (DrainRxLocked()) {}
+        g_rxTail.store(g_rxHead.load(std::memory_order_relaxed), std::memory_order_release);
+        g_hwLock.Release();
+
+        if (!baudOk) {
+            KernelLogStream(ERROR, "BtUart") << "115200 baud unreachable from a "
+                << (uint64_t)g_clockHz << " Hz clock";
+            return false;
+        }
+
+        if (config.gsi != 0) {
+            uint8_t line = Hal::GsiRouting::LineForGsi(config.gsi, config.levelTriggered,
+                                                       config.activeLow);
+            if (line != 0) {
+                Arch::RegisterIrqHandler(line, HandleInterrupt);
+                g_interrupt = true;
+                g_hwLock.Acquire();
+                if (g_kind == Kind::MiniUart) {
+                    Write32(MU_IER, MU_IER_RX | MU_IER_REQUIRED);
+                } else {
+                    Write32(PL_IMSC, PL_INT_RX | PL_INT_RT | PL_INT_OE);
+                }
+                g_hwLock.Release();
+                Arch::EnableLegacyIrq(line);
+            }
+        }
+
+        g_initialized = true;
+        KernelLogStream(OK, "BtUart") << (g_kind == Kind::MiniUart ? "Mini UART" : "PL011")
+            << " at 0x" << base::hex << base << base::dec << ", " << (uint64_t)g_clockHz
+            << " Hz clock, " << (g_interrupt ? "interrupt driven" : "polled (no interrupt)");
+        return true;
+    }
+
+    bool IsInitialized() { return g_initialized; }
+    bool HasInterrupt() { return g_interrupt; }
+    uint32_t GetBaud() { return g_baud; }
+    Kind GetKind() { return g_kind; }
+    bool FlowControl() { return g_flowControl; }
+
+    bool BaudReachable(uint32_t baud) {
+        uint64_t divisor = 0;
+        return g_initialized && ComputeDivisor(baud, divisor);
+    }
+
+    bool SetBaud(uint32_t baud) {
+        if (!g_initialized) return false;
+        WaitTxIdle(1000);
+        g_hwLock.Acquire();
+        bool ok = ProgramBaudLocked(baud);
+        g_hwLock.Release();
+        return ok;
+    }
+
+    void SetFlowControl(bool on) {
+        if (!g_initialized) return;
+        g_hwLock.Acquire();
+        if (g_kind == Kind::MiniUart) {
+            uint32_t cntl = MU_CNTL_RXEN | MU_CNTL_TXEN
+                          | (on ? MU_CNTL_AUTORTS | MU_CNTL_RTS_LOW
+                                | MU_CNTL_AUTOCTS | MU_CNTL_CTS_LOW : 0);
+            Write32(MU_CNTL, cntl);
+            // With auto RTS off, hold RTS asserted by hand.
+            Write32(MU_MCR, MU_MCR_RTS);
+        } else {
+            uint32_t cr = Read32(PL_CR) & ~(PL_CR_RTSEN | PL_CR_CTSEN);
+            if (on) cr |= PL_CR_RTSEN | PL_CR_CTSEN;
+            Write32(PL_CR, cr | PL_CR_RTS);
+        }
+        g_flowControl = on;
+        g_hwLock.Release();
+    }
+
+    void DeassertRts() {
+        if (!g_initialized) return;
+        g_hwLock.Acquire();
+        if (g_kind == Kind::MiniUart) {
+            Write32(MU_CNTL, MU_CNTL_RXEN | MU_CNTL_TXEN);
+            Write32(MU_MCR, 0);
+        } else {
+            Write32(PL_CR, Read32(PL_CR) & ~(PL_CR_RTSEN | PL_CR_CTSEN | PL_CR_RTS));
+        }
+        g_flowControl = false;
+        g_hwLock.Release();
+    }
+
+    int Read(uint8_t* buffer, int maxLength) {
+        if (buffer == nullptr || maxLength <= 0) return 0;
+        uint32_t tail = g_rxTail.load(std::memory_order_relaxed);
+        uint32_t head = g_rxHead.load(std::memory_order_acquire);
+        uint32_t available = head - tail;
+        uint32_t n = available < (uint32_t)maxLength ? available : (uint32_t)maxLength;
+        for (uint32_t i = 0; i < n; i++) {
+            buffer[i] = g_rxRing[(tail + i) & (RX_RING_SIZE - 1)];
+        }
+        g_rxTail.store(tail + n, std::memory_order_release);
+        return (int)n;
+    }
+
+    bool HasRxData() {
+        if (!g_initialized) return false;
+        if (g_rxHead.load(std::memory_order_acquire) != g_rxTail.load(std::memory_order_relaxed))
+            return true;
+        // Polled: nothing fills the ring until Service() runs, so look at
+        // the FIFO itself.
+        if (g_interrupt) return false;
+        return g_kind == Kind::MiniUart ? (Read32(MU_LSR) & MU_LSR_DATA_READY) != 0
+                                        : (Read32(PL_FR) & PL_FR_RXFE) == 0;
+    }
+
+    void FlushRx() {
+        if (!g_initialized) return;
+        g_hwLock.Acquire();
+        while (DrainRxLocked()) {}
+        g_rxTail.store(g_rxHead.load(std::memory_order_relaxed), std::memory_order_release);
+        g_hwLock.Release();
+    }
+
+    bool Write(const uint8_t* head, uint32_t headLength,
+               const uint8_t* body, uint32_t bodyLength) {
+        if (body == nullptr) bodyLength = 0;
+        uint32_t length = headLength + bodyLength;
+        if (!g_initialized || length == 0) return length == 0;
+        if (length > TX_RING_SIZE) return false;
+        g_hwLock.Acquire();
+        if (TX_RING_SIZE - (g_txHead - g_txTail) < length) {
+            g_hwLock.Release();
+            return false;
+        }
+        for (uint32_t i = 0; i < headLength; i++) {
+            g_txRing[(g_txHead + i) & (TX_RING_SIZE - 1)] = head[i];
+        }
+        for (uint32_t i = 0; i < bodyLength; i++) {
+            g_txRing[(g_txHead + headLength + i) & (TX_RING_SIZE - 1)] = body[i];
+        }
+        g_txHead += length;
+        FillTxLocked();
+        g_hwLock.Release();
+        return true;
+    }
+
+    uint32_t TxFree() {
+        return TX_RING_SIZE - g_txCount.load(std::memory_order_acquire);
+    }
+
+    uint32_t TxPending() {
+        return g_txCount.load(std::memory_order_acquire);
+    }
+
+    bool WaitTxIdle(uint32_t timeoutMs) {
+        if (!g_initialized) return false;
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            g_hwLock.Acquire();
+            FillTxLocked();
+            bool idle = TransmitterIdleLocked();
+            g_hwLock.Release();
+            if (idle) return true;
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
+            Arch::Pause();
+        }
+    }
+
+    void Service() {
+        if (!g_initialized) return;
+        g_hwLock.Acquire();
+        DrainRxLocked();
+        FillTxLocked();
+        g_hwLock.Release();
+    }
+
+    void GetStats(Stats& out) {
+        g_hwLock.Acquire();
+        out = g_stats;
+        g_hwLock.Release();
+    }
+
+}

kernel/src/Drivers/Serial/BtUart.hpp

@@ -0,0 +1,96 @@
+/*
+    * BtUart.hpp
+    * Interrupt-driven UART to an on-board Bluetooth controller
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// A byte pipe to a peripheral wired to one of the Broadcom SoC's UARTs --
+// on the Raspberry Pi 4/400 the Bluetooth half of the CYW4345x radio.
+// Unlike the console (Uart.cpp: polled, transmit only) this one receives,
+// so it is interrupt driven: the hard IRQ moves bytes between the FIFOs and
+// two software rings, and everything else runs in process context.
+//
+// Two UARTs can carry it:
+//   - the mini UART (BCM2835 AUX, 8-byte FIFOs, baud from the VPU core
+//     clock), which is where Bluetooth sits with the miniuart-bt overlay
+//     the Pi firmware task force's UEFI ships;
+//   - the PL011 (32-byte FIFOs, fixed 48 MHz reference clock).
+// Pin routing (GPIO 30-33) is the caller's job.
+
+namespace Drivers::Serial::BtUart {
+
+    enum class Kind : uint8_t {
+        MiniUart,
+        Pl011,
+    };
+
+    struct Config {
+        Kind     kind;
+        uint64_t mmioPhys;        // the block ACPI describes (mini UART: the AUX block)
+        uint32_t clockHz;         // UART reference clock
+        uint32_t gsi;             // 0 = no interrupt, poll from Service()
+        bool     levelTriggered;
+        bool     activeLow;
+    };
+
+    // Program the UART to 115200 8N1 with RTS asserted and start receiving.
+    bool Initialize(const Config& config);
+    bool IsInitialized();
+
+    // Whether the receive path is interrupt driven. Without it, bytes only
+    // move when Service() runs, so a caller should keep the baud rate low.
+    bool HasInterrupt();
+
+    // Change the baud rate. Waits for queued output to drain first, since
+    // the peer is about to switch too. Returns false if the rate is
+    // unreachable from the reference clock (more than 2.5% off).
+    bool SetBaud(uint32_t baud);
+    bool BaudReachable(uint32_t baud);
+    uint32_t GetBaud();
+    Kind GetKind();
+
+    // Hardware flow control: deassert RTS while the receive FIFO is nearly
+    // full so the peer pauses instead of overrunning it, and hold our
+    // transmitter while the peer deasserts CTS.
+    void SetFlowControl(bool on);
+    bool FlowControl();
+
+    // Drive RTS deasserted (and stop auto RTS) until the next
+    // SetFlowControl. The peer may sample it at power-on: a Broadcom
+    // controller that sees its CTS asserted then boots into autobaud mode.
+    void DeassertRts();
+
+    // Receive ring (single consumer). Read returns the number of bytes copied.
+    int  Read(uint8_t* buffer, int maxLength);
+    bool HasRxData();
+    void FlushRx();
+
+    // Queue bytes for transmission: all of them, or none (false) when the
+    // transmit ring lacks room. Safe from several producers at once; each
+    // call's bytes (head, then body) stay contiguous on the wire.
+    bool Write(const uint8_t* head, uint32_t headLength,
+               const uint8_t* body = nullptr, uint32_t bodyLength = 0);
+    uint32_t TxFree();
+    uint32_t TxPending();
+
+    // Wait until everything queued has left the transmitter.
+    bool WaitTxIdle(uint32_t timeoutMs);
+
+    // Move bytes between the FIFOs and the rings from process context: the
+    // whole receive path when there is no interrupt, and a safety net when
+    // there is one.
+    void Service();
+
+    struct Stats {
+        uint64_t rxBytes;
+        uint64_t txBytes;
+        uint64_t rxDropped;       // receive ring full
+        uint64_t overruns;        // receive FIFO overflowed in hardware
+        uint64_t interrupts;
+    };
+    void GetStats(Stats& out);
+
+}

kernel/src/Drivers/Serial/Uart.cpp

@@ -0,0 +1,187 @@
+/*
+    * Uart.cpp
+    * Polled serial console from the ACPI SPCR table
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Uart.hpp"
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Io/IoPort.hpp>
+#include <Arch/Cpu.hpp>
+
+using namespace Kt;
+
+namespace Drivers::Serial {
+
+    struct GenericAddress {
+        uint8_t  AddressSpaceId;  // 0 = system memory, 1 = system I/O
+        uint8_t  RegisterBitWidth;
+        uint8_t  RegisterBitOffset;
+        uint8_t  AccessSize;      // 1 = byte, 2 = word, 3 = dword, 4 = qword
+        uint64_t Address;
+    } __attribute__((packed));
+
+    struct Spcr {
+        Hal::ACPI::CommonSDTHeader Header;
+        uint8_t        InterfaceType;
+        uint8_t        Reserved[3];
+        GenericAddress BaseAddress;
+        uint8_t        InterruptType;
+        uint8_t        Irq;
+        uint32_t       GlobalSystemInterrupt;
+        uint8_t        BaudRate;
+        uint8_t        Parity;
+        uint8_t        StopBits;
+        uint8_t        FlowControl;
+        uint8_t        TerminalType;
+    } __attribute__((packed));
+
+    // SPCR / DBG2 serial interface subtypes
+    static constexpr uint8_t Type16550        = 0x00;
+    static constexpr uint8_t Type16450        = 0x01;
+    static constexpr uint8_t TypePl011        = 0x03;
+    static constexpr uint8_t TypeSbsa32       = 0x0D;
+    static constexpr uint8_t TypeSbsaGeneric  = 0x0E;
+    static constexpr uint8_t TypeBcm2835      = 0x10;
+    static constexpr uint8_t Type16550Generic = 0x12;
+
+    // PL011: data register, flag register (TX FIFO full)
+    static constexpr uint64_t Pl011Dr = 0x00;
+    static constexpr uint64_t Pl011Fr = 0x18;
+    static constexpr uint32_t Pl011FrTxff = 1u << 5;
+
+    // 16550: transmit holding register, line status register (THR empty)
+    static constexpr uint64_t Uart16550Thr = 0;
+    static constexpr uint64_t Uart16550Lsr = 5;
+    static constexpr uint8_t Uart16550LsrThre = 1u << 5;
+
+    // Bounded so a wedged or absent UART cannot hang the console.
+    static constexpr int TxSpinLimit = 100000;
+
+    static volatile uint8_t* g_mmio = nullptr;
+    static uint16_t g_port = 0;
+    static uint32_t g_stride = 1;
+    static uint64_t g_consoleAddress = 0;
+
+    static inline uint32_t Read32(uint64_t offset) {
+        return *(volatile uint32_t*)(g_mmio + offset);
+    }
+
+    static inline void Write32(uint64_t offset, uint32_t value) {
+        *(volatile uint32_t*)(g_mmio + offset) = value;
+    }
+
+    static void Pl011Putchar(char c) {
+        for (int i = 0; i < TxSpinLimit && (Read32(Pl011Fr) & Pl011FrTxff); i++) {
+            Arch::Pause();
+        }
+        Write32(Pl011Dr, (uint8_t)c);
+    }
+
+    static uint8_t Mmio16550Read(uint64_t reg) {
+        if (g_stride == 4) return (uint8_t)Read32(reg * 4);
+        return g_mmio[reg];
+    }
+
+    static void Mmio16550Write(uint64_t reg, uint8_t value) {
+        if (g_stride == 4) Write32(reg * 4, value);
+        else g_mmio[reg] = value;
+    }
+
+    static void Mmio16550Putchar(char c) {
+        for (int i = 0; i < TxSpinLimit && !(Mmio16550Read(Uart16550Lsr) & Uart16550LsrThre); i++) {
+            Arch::Pause();
+        }
+        Mmio16550Write(Uart16550Thr, (uint8_t)c);
+    }
+
+    static void Port16550Putchar(char c) {
+        for (int i = 0; i < TxSpinLimit && !(Io::In8(g_port + Uart16550Lsr) & Uart16550LsrThre); i++) {
+            Arch::Pause();
+        }
+        Io::Out8((uint8_t)c, g_port + Uart16550Thr);
+    }
+
+    static Spcr* FindSpcr(Hal::ACPI::CommonSDTHeader* xsdt) {
+        uint32_t entryCount = (xsdt->Length - sizeof(Hal::ACPI::CommonSDTHeader)) / 8;
+        uint64_t* entries = (uint64_t*)((uint64_t)xsdt + sizeof(Hal::ACPI::CommonSDTHeader));
+
+        for (uint32_t i = 0; i < entryCount; i++) {
+            auto* header = (Hal::ACPI::CommonSDTHeader*)Memory::HHDM(entries[i]);
+            const char* sig = header->Signature;
+            if (sig[0] == 'S' && sig[1] == 'P' && sig[2] == 'C' && sig[3] == 'R') {
+                return (Spcr*)header;
+            }
+        }
+        return nullptr;
+    }
+
+    static volatile uint8_t* MapRegisters(uint64_t phys) {
+        uint64_t page = phys & ~0xFFFULL;
+        Memory::VMM::g_paging->MapMMIO(page, Memory::HHDM(page));
+        return (volatile uint8_t*)Memory::HHDM(phys);
+    }
+
+    void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt) {
+        if (xsdt == nullptr) return;
+
+        Spcr* spcr = FindSpcr(xsdt);
+        if (spcr == nullptr || spcr->Header.Length < sizeof(Spcr)) {
+            return;
+        }
+
+        const GenericAddress& reg = spcr->BaseAddress;
+        if (reg.Address == 0) return;
+
+        SerialSink sink = nullptr;
+        const char* name = "";
+        bool isIo = reg.AddressSpaceId == 1;
+
+        switch (spcr->InterfaceType) {
+            case TypePl011:
+            case TypeSbsa32:
+            case TypeSbsaGeneric:
+                if (isIo) return;
+                g_mmio = MapRegisters(reg.Address);
+                sink = Pl011Putchar;
+                name = "PL011";
+                break;
+
+            case Type16550:
+            case Type16450:
+            case Type16550Generic:
+            case TypeBcm2835:
+                if (isIo) {
+                    if (!Io::HasPortIo) return;
+                    g_port = (uint16_t)reg.Address;
+                    sink = Port16550Putchar;
+                } else {
+                    // The BCM2835 mini UART is a 16550 with 32-bit registers.
+                    g_stride = (reg.RegisterBitWidth == 32 || reg.AccessSize == 3
+                                || spcr->InterfaceType == TypeBcm2835) ? 4 : 1;
+                    g_mmio = MapRegisters(reg.Address);
+                    sink = Mmio16550Putchar;
+                }
+                name = spcr->InterfaceType == TypeBcm2835 ? "BCM2835 mini UART" : "16550";
+                break;
+
+            default:
+                KernelLogStream(WARNING, "Serial") << "SPCR interface type 0x" << base::hex
+                    << (uint64_t)spcr->InterfaceType << " not supported";
+                return;
+        }
+
+        g_consoleAddress = reg.Address;
+        AttachSerialConsole(sink);
+        KernelLogStream(OK, "Serial") << "Console on " << name << " at " << (isIo ? "port " : "")
+            << "0x" << base::hex << reg.Address;
+    }
+
+    uint64_t ConsoleAddress() {
+        return g_consoleAddress;
+    }
+
+}

kernel/src/Drivers/Serial/Uart.hpp

@@ -0,0 +1,22 @@
+/*
+    * Uart.hpp
+    * Polled serial console from the ACPI SPCR table
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <ACPI/ACPI.hpp>
+
+namespace Drivers::Serial {
+
+    // Find the firmware's serial console (SPCR: PL011, SBSA generic UART,
+    // 16550 or the BCM2835 mini UART) and mirror the kernel console to it.
+    // Uses the baud rate the firmware configured. Needs the kernel page
+    // tables (the registers are mapped uncached through the HHDM).
+    void InitializeConsole(Hal::ACPI::CommonSDTHeader* xsdt);
+
+    // Physical address of the console UART's registers, or 0 when there is
+    // none (so another driver does not take over the console's UART).
+    uint64_t ConsoleAddress();
+
+}

kernel/src/Drivers/Storage/Ahci.cpp

@@ -267,13 +267,15 @@ namespace Drivers::Storage::Ahci {
             }
         }
 
-        // Issue the command
+        // Issue the command once its header, table and PRDs are visible
+        Arch::DmaWriteFence();
         WritePortReg(port, PORT_CI, 1u << slot);
 
         // Wait for completion
         for (int i = 0; i < 5000000; i++) {
             uint32_t ci = ReadPortReg(port, PORT_CI);
             if (!(ci & (1u << slot))) {
+                Arch::DmaReadFence();
                 // Check for errors
                 uint32_t is = ReadPortReg(port, PORT_IS);
                 if (is & PORT_IS_TFES) {
@@ -641,10 +643,19 @@ namespace Drivers::Storage::Ahci {
         for (int i = 0; i < MAX_PORTS; i++) {
             if (!(g_portsImplemented & (1u << i))) continue;
 
+            // The signature arrives in a D2H FIS, which an HBA may only
+            // latch into PxSIG once FIS receive is running (QEMU does so):
+            // start every port with a device on its link, then classify.
+            if ((ReadPortReg(i, PORT_SSTS) & SSTS_DET_MASK) != SSTS_DET_PRESENT) continue;
+            InitPort(i);
+
             PortType type = ClassifyPort(i);
             g_ports[i].Type = type;
 
-            if (type == PortType::None) continue;
+            if (type == PortType::None) {
+                StopPort(i);
+                continue;
+            }
 
             const char* typeStr = "Unknown";
             switch (type) {
@@ -658,8 +669,6 @@ namespace Drivers::Storage::Ahci {
             KernelLogStream(INFO, "AHCI") << "Port " << base::dec << (uint64_t)i
                 << ": " << typeStr << " device detected";
 
-            InitPort(i);
-
             if (type == PortType::Sata) {
                 if (IdentifyDevice(i)) {
                     g_ports[i].Active = true;

kernel/src/Drivers/Storage/BlockDevice.hpp

@@ -17,6 +17,7 @@ namespace Drivers::Storage {
         BLOCK_KIND_SATAPI  = 2,
         BLOCK_KIND_NVME    = 3,
         BLOCK_KIND_USB_MSC = 4,
+        BLOCK_KIND_SD      = 5,
     };
 
     struct BlockDevice {

kernel/src/Drivers/Storage/Gpt.cpp

@@ -107,6 +107,8 @@ namespace Drivers::Storage::Gpt {
                 return GUID_LINUX_SWAP;
             case 0x83:
                 return GUID_LINUX_FS;
+            case 0xEF:
+                return GUID_EFI_SYSTEM;
             default:
                 return GUID_BASIC_DATA;
         }
@@ -551,6 +553,78 @@ namespace Drivers::Storage::Gpt {
         return removed;
     }
 
+    // -------------------------------------------------------------------------
+    // MBR partition resize
+    // -------------------------------------------------------------------------
+
+    // Read sector 0 into mbr and return the primary slot that holds `part`, or
+    // -1 when the device is GPT (protective MBR) or `part` is not a primary
+    // entry (a logical partition inside an extended one).
+    static int FindMbrSlot(const PartitionInfo& part, ProtectiveMbr& mbr) {
+        const BlockDevice* dev = GetBlockDevice(part.BlockDevIndex);
+        if (!dev || !dev->ReadSectors(dev->Ctx, 0, 1, &mbr)) return -1;
+        if (mbr.Signature != 0xAA55 || ValidateProtectiveMbr((const uint8_t*)&mbr)) return -1;
+
+        for (int i = 0; i < 4; i++) {
+            const MbrPartitionEntry& e = mbr.Partitions[i];
+            if (e.Type == 0 || IsExtendedMbrType(e.Type)) continue;
+            if (e.LbaFirst == part.StartLba && e.SectorCount == part.SectorCount) return i;
+        }
+        return -1;
+    }
+
+    uint64_t GetMbrPartitionMaxSectors(int index) {
+        const PartitionInfo* part = GetPartition(index);
+        if (!part) return 0;
+
+        ProtectiveMbr mbr;
+        if (FindMbrSlot(*part, mbr) < 0) return 0;
+        const BlockDevice* dev = GetBlockDevice(part->BlockDevIndex);
+
+        // Stop at whichever primary entry starts next (an extended entry covers
+        // every logical partition inside it), else at the end of the device.
+        uint64_t end = dev->SectorCount;
+        for (int i = 0; i < 4; i++) {
+            const MbrPartitionEntry& e = mbr.Partitions[i];
+            if (e.Type == 0 || e.SectorCount == 0) continue;
+            if (e.LbaFirst > part->StartLba && e.LbaFirst < end) end = e.LbaFirst;
+        }
+
+        // The entry stores a 32-bit sector count; keep the end addressable too.
+        if (end > 0xFFFFFFFFull) end = 0xFFFFFFFFull;
+        return end > part->StartLba ? end - part->StartLba : 0;
+    }
+
+    int ResizeMbrPartition(int index, uint64_t sectorCount) {
+        if (index < 0 || index >= g_partitionCount) return -1;
+        PartitionInfo& part = g_partitions[index];
+        if (sectorCount < part.SectorCount) return -1;
+        if (sectorCount > GetMbrPartitionMaxSectors(index)) return -1;
+        if (sectorCount == part.SectorCount) return 0;
+
+        ProtectiveMbr mbr;
+        int slot = FindMbrSlot(part, mbr);
+        if (slot < 0) return -1;
+
+        MbrPartitionEntry& e = mbr.Partitions[slot];
+        e.SectorCount = (uint32_t)sectorCount;
+        // CHS cannot address past 8 GiB; FE FF FF tells CHS-era tools to use
+        // the LBA fields instead.
+        e.ChsLast[0] = 0xFE;
+        e.ChsLast[1] = 0xFF;
+        e.ChsLast[2] = 0xFF;
+
+        const BlockDevice* dev = GetBlockDevice(part.BlockDevIndex);
+        if (!dev->WriteSectors(dev->Ctx, 0, 1, &mbr)) return -1;
+        if (dev->Flush) dev->Flush(dev->Ctx);
+
+        KernelLogStream(OK, "MBR") << "Partition " << index << " grown from "
+            << part.SectorCount << " to " << sectorCount << " sectors";
+        part.SectorCount = sectorCount;
+        part.EndLba = part.StartLba + sectorCount - 1;
+        return 0;
+    }
+
     // -------------------------------------------------------------------------
     // ASCII to UTF-16LE for partition names
     // -------------------------------------------------------------------------

kernel/src/Drivers/Storage/Gpt.hpp

@@ -127,6 +127,18 @@ namespace Drivers::Storage::Gpt {
     // hot-pluggable block device disappears.
     int RemovePartitionsForBlockDevice(int blockDevIndex);
 
+    // Largest size, in sectors, that partition `index` could be grown to: up to
+    // the next partition on its device or the end of the device, within the
+    // 32-bit LBA fields of an MBR entry. Returns 0 if the partition is not a
+    // primary entry of a plain MBR disk (GPT disks and logical partitions are
+    // not resized).
+    uint64_t GetMbrPartitionMaxSectors(int index);
+
+    // Grow MBR partition `index` in place to sectorCount sectors, rewriting its
+    // table entry and the in-memory record. Data is untouched. Returns 0 on
+    // success, -1 if the partition cannot be resized to that size.
+    int ResizeMbrPartition(int index, uint64_t sectorCount);
+
     // Look up the human-readable name for a partition type GUID.
     const char* GetTypeName(const Guid& typeGuid);
 

kernel/src/Drivers/Storage/Nvme.cpp

@@ -87,7 +87,9 @@ namespace Drivers::Storage::Nvme {
     //            CQ y Head Doorbell offset = 0x1000 + ((2y+1) * doorbellStride)
     // -------------------------------------------------------------------------
 
+    // The submission queue entry must be visible before the doorbell.
     static void WriteSqTailDoorbell(uint16_t queueId, uint16_t value) {
+        Arch::DmaWriteFence();
         uint32_t offset = 0x1000 + (2 * queueId) * g_doorbellStride;
         *(volatile uint32_t*)(g_mmioBase + offset) = value;
     }
@@ -152,6 +154,8 @@ namespace Drivers::Storage::Nvme {
             uint16_t status = cqe->Status;
 
             if ((status & CQE_PHASE_BIT) == phase) {
+                // The rest of the entry (and the data) before the phase bit.
+                Arch::DmaReadFence();
                 uint16_t consumedId = cqe->CommandId;
                 uint32_t result = cqe->Result;
                 uint16_t sqHead = cqe->SqHead;

kernel/src/Drivers/Storage/Sdhci.cpp

@@ -0,0 +1,410 @@
+/*
+    * Sdhci.cpp
+    * SD Host Controller (SDHCI) driver: SD cards as block devices
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Sdhci.hpp"
+#include "BlockDevice.hpp"
+#include <Arch/Cpu.hpp>
+#include <CppLib/Spinlock.hpp>
+#include <CppLib/Stream.hpp>
+#include <Libraries/Memory.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// One slot, polled, programmed I/O through the buffer data port. PIO keeps
+// the driver free of DMA address translation and cache coherency, which on
+// the Raspberry Pi 4 depend on the SoC stepping (EMMC2's _DMA).
+//
+// Every register access is 32 bits wide: the Broadcom controllers (the Pi's
+// EMMC2, and the Arasan-derived EMMC before it) do not support narrower
+// ones. 8/16-bit fields are written by read-modify-write of their dword,
+// and the transfer-mode/command pair in a single write that issues the
+// command.
+
+using namespace Kt;
+
+namespace Drivers::Storage::Sdhci {
+
+    // Registers (SD Host Controller Simplified Specification 3.00)
+    static constexpr uint32_t REG_BLOCK_SIZE_COUNT = 0x04;  // size [15:0], count [31:16]
+    static constexpr uint32_t REG_ARGUMENT         = 0x08;
+    static constexpr uint32_t REG_XFER_MODE_CMD    = 0x0C;  // transfer mode [15:0], command [31:16]
+    static constexpr uint32_t REG_RESPONSE         = 0x10;  // 4 dwords
+    static constexpr uint32_t REG_BUFFER_DATA      = 0x20;
+    static constexpr uint32_t REG_PRESENT_STATE    = 0x24;
+    static constexpr uint32_t REG_HOST_CONTROL     = 0x28;  // host ctl 1 [7:0], power [15:8]
+    static constexpr uint32_t REG_CLOCK_CONTROL    = 0x2C;  // clock [15:0], timeout [23:16], reset [31:24]
+    static constexpr uint32_t REG_INT_STATUS       = 0x30;  // normal [15:0], error [31:16]
+    static constexpr uint32_t REG_INT_ENABLE       = 0x34;  // status enables
+    static constexpr uint32_t REG_INT_SIGNAL       = 0x38;  // interrupt signal enables
+    static constexpr uint32_t REG_CAPABILITIES     = 0x40;
+    static constexpr uint32_t REG_VERSION          = 0xFC;  // host version [31:16]
+
+    // Present state
+    static constexpr uint32_t PS_CMD_INHIBIT = 1u << 0;
+    static constexpr uint32_t PS_DAT_INHIBIT = 1u << 1;
+
+    // Interrupt status
+    static constexpr uint32_t INT_CMD_COMPLETE  = 1u << 0;
+    static constexpr uint32_t INT_XFER_COMPLETE = 1u << 1;
+    static constexpr uint32_t INT_WRITE_READY   = 1u << 4;
+    static constexpr uint32_t INT_READ_READY    = 1u << 5;
+    static constexpr uint32_t INT_ERROR         = 1u << 15;
+    static constexpr uint32_t INT_ERROR_MASK    = 0xFFFF0000u;
+
+    // Host control 1 / power control
+    static constexpr uint32_t HC_4BIT       = 1u << 1;
+    static constexpr uint32_t PWR_ON        = 1u << 8;
+    static constexpr uint32_t PWR_330       = 7u << 9;
+    static constexpr uint32_t PWR_300       = 6u << 9;
+    static constexpr uint32_t PWR_180       = 5u << 9;
+
+    // Clock control
+    static constexpr uint32_t CLK_INT_EN     = 1u << 0;
+    static constexpr uint32_t CLK_INT_STABLE = 1u << 1;
+    static constexpr uint32_t CLK_SD_EN      = 1u << 2;
+    static constexpr uint32_t TIMEOUT_MAX    = 0xEu << 16;
+
+    // Software reset (byte 0x2F)
+    static constexpr uint32_t RESET_ALL = 1u << 24;
+    static constexpr uint32_t RESET_CMD = 1u << 25;
+    static constexpr uint32_t RESET_DAT = 1u << 26;
+
+    // Capabilities
+    static constexpr uint32_t CAP_VOLT_330 = 1u << 24;
+    static constexpr uint32_t CAP_VOLT_300 = 1u << 25;
+
+    // Transfer mode
+    static constexpr uint32_t XM_BLOCK_COUNT_EN = 1u << 1;
+    static constexpr uint32_t XM_AUTO_CMD12     = 1u << 2;
+    static constexpr uint32_t XM_READ           = 1u << 4;
+    static constexpr uint32_t XM_MULTI_BLOCK    = 1u << 5;
+
+    // Command register flags
+    enum Response : uint32_t {
+        RespNone  = 0,
+        Resp136   = 1,            // R2
+        Resp48    = 2 | 3u << 3,  // R1, R6, R7: CRC and index checked
+        Resp48Busy= 3 | 3u << 3,  // R1b
+        Resp48NoCrc = 2,          // R3
+    };
+    static constexpr uint32_t CMD_DATA = 1u << 5;
+
+    static constexpr uint32_t BlockSize = 512;
+    // The block count register is 16 bits wide.
+    static constexpr uint32_t MaxBlocksPerCommand = 0xFFFF;
+    static constexpr uint64_t CommandTimeoutMs = 200;
+    static constexpr uint64_t DataTimeoutMs = 1000;
+
+    // Base clock when the capabilities leave it to the platform: the
+    // Raspberry Pi 4's EMMC2 runs from a 100 MHz clock the firmware sets up.
+    static constexpr uint32_t FallbackBaseClockHz = 100000000;
+
+    static volatile uint8_t* g_regs = nullptr;
+    static bool g_initialized = false;
+    static uint32_t g_version = 0;      // 0: 1.00, 1: 2.00, 2: 3.00
+    static uint32_t g_baseClockHz = 0;
+    static uint32_t g_rca = 0;
+    static bool g_highCapacity = false;
+    static uint64_t g_sectorCount = 0;
+    static kcp::Mutex g_lock;
+
+    static inline uint32_t Read(uint32_t reg) {
+        return *(volatile uint32_t*)(g_regs + reg);
+    }
+
+    static inline void Write(uint32_t reg, uint32_t value) {
+        *(volatile uint32_t*)(g_regs + reg) = value;
+    }
+
+    static bool WaitClear(uint32_t reg, uint32_t mask, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Read(reg) & mask) {
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
+            Arch::Pause();
+        }
+        return true;
+    }
+
+    static bool Reset(uint32_t which) {
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | which);
+        return WaitClear(REG_CLOCK_CONTROL, which, 100);
+    }
+
+    // Wait for any of `bits` in the normal status, or an error. Clears what
+    // it returns on.
+    static bool WaitStatus(uint32_t bits, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            uint32_t status = Read(REG_INT_STATUS);
+            if (status & (INT_ERROR | INT_ERROR_MASK)) {
+                Write(REG_INT_STATUS, status);
+                Reset(RESET_CMD | RESET_DAT);
+                return false;
+            }
+            if (status & bits) {
+                Write(REG_INT_STATUS, status & bits);
+                return true;
+            }
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) {
+                Reset(RESET_CMD | RESET_DAT);
+                return false;
+            }
+            Arch::Pause();
+        }
+    }
+
+    static bool SendCommand(uint32_t index, uint32_t argument, uint32_t response,
+                            uint32_t transferMode = 0, bool data = false) {
+        uint32_t inhibit = PS_CMD_INHIBIT;
+        if (data || response == Resp48Busy) inhibit |= PS_DAT_INHIBIT;
+        if (!WaitClear(REG_PRESENT_STATE, inhibit, CommandTimeoutMs)) {
+            Reset(RESET_CMD | RESET_DAT);
+            return false;
+        }
+
+        Write(REG_INT_STATUS, 0xFFFFFFFF);
+        Write(REG_ARGUMENT, argument);
+        uint32_t command = index << 8 | response | (data ? CMD_DATA : 0);
+        Write(REG_XFER_MODE_CMD, command << 16 | transferMode);
+
+        if (!WaitStatus(INT_CMD_COMPLETE, CommandTimeoutMs)) return false;
+        if (response == Resp48Busy && !WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs)) return false;
+        return true;
+    }
+
+    // An application command: CMD55 with the card's address first.
+    static bool SendAppCommand(uint32_t index, uint32_t argument, uint32_t response) {
+        return SendCommand(55, g_rca << 16, Resp48) && SendCommand(index, argument, response);
+    }
+
+    // Bits [start, start+length) of a 136-bit response (CID/CSD numbering).
+    // The controller drops the CRC byte, so CSD bit n is response bit n-8.
+    static uint32_t ResponseBits(const uint32_t response[4], int start, int length) {
+        uint32_t value = 0;
+        for (int i = 0; i < length; i++) {
+            int bit = start + i - 8;
+            if (response[bit / 32] & (1u << (bit % 32))) value |= 1u << i;
+        }
+        return value;
+    }
+
+    static bool SetClock(uint32_t hz) {
+        uint32_t clock = Read(REG_CLOCK_CONTROL);
+        Write(REG_CLOCK_CONTROL, clock & ~(0xFFFFu));
+
+        uint32_t divider = 0;   // base clock / (2 * divider); 0 = undivided
+        if (hz < g_baseClockHz) {
+            divider = (g_baseClockHz + 2 * hz - 1) / (2 * hz);
+            uint32_t max = g_version >= 2 ? 0x3FF : 0x80;
+            if (divider > max) divider = max;
+            if (g_version < 2) {
+                // Before 3.00 the divider is a power of two.
+                uint32_t p = 1;
+                while (p < divider) p <<= 1;
+                divider = p;
+            }
+        }
+
+        uint32_t field = (divider & 0xFF) << 8 | ((divider >> 8) & 0x3) << 6;
+        clock = (Read(REG_CLOCK_CONTROL) & ~0xFFFFu) | field | CLK_INT_EN;
+        Write(REG_CLOCK_CONTROL, clock);
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (!(Read(REG_CLOCK_CONTROL) & CLK_INT_STABLE)) {
+            if (Timekeeping::GetMilliseconds() - start > 20) return false;
+            Arch::Pause();
+        }
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | CLK_SD_EN);
+        return true;
+    }
+
+    static bool InitializeCard() {
+        uint32_t response[4];
+
+        if (!SendCommand(0, 0, RespNone)) return false;
+
+        // CMD8: a 2.00+ card echoes the check pattern.
+        bool v2 = SendCommand(8, 0x1AA, Resp48) && (Read(REG_RESPONSE) & 0xFFF) == 0x1AA;
+
+        // ACMD41 until the card leaves its busy state (up to a second).
+        uint32_t ocr = 0;
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            if (!SendAppCommand(41, (v2 ? 1u << 30 : 0) | 0x00FF8000, Resp48NoCrc)) return false;
+            ocr = Read(REG_RESPONSE);
+            if (ocr & (1u << 31)) break;
+            if (Timekeeping::GetMilliseconds() - start > 1000) return false;
+        }
+        g_highCapacity = (ocr & (1u << 30)) != 0;
+
+        if (!SendCommand(2, 0, Resp136)) return false;
+        if (!SendCommand(3, 0, Resp48)) return false;
+        g_rca = Read(REG_RESPONSE) >> 16;
+
+        if (!SendCommand(9, g_rca << 16, Resp136)) return false;
+        for (int i = 0; i < 4; i++) response[i] = Read(REG_RESPONSE + i * 4);
+        if (ResponseBits(response, 126, 2) == 1) {
+            uint64_t cSize = ResponseBits(response, 48, 22);
+            g_sectorCount = (cSize + 1) * 1024;
+        } else {
+            uint64_t cSize = ResponseBits(response, 62, 12);
+            uint64_t mult = ResponseBits(response, 47, 3);
+            uint64_t readBlockLength = ResponseBits(response, 80, 4);
+            g_sectorCount = ((cSize + 1) << (mult + 2)) * (1ULL << readBlockLength) / BlockSize;
+        }
+
+        if (!SendCommand(7, g_rca << 16, Resp48Busy)) return false;
+        if (!g_highCapacity && !SendCommand(16, BlockSize, Resp48)) return false;
+
+        // 4-bit bus when the card agrees; 1-bit otherwise.
+        if (SendAppCommand(6, 2, Resp48)) {
+            Write(REG_HOST_CONTROL, Read(REG_HOST_CONTROL) | HC_4BIT);
+        }
+
+        SetClock(25000000);
+        return true;
+    }
+
+    // Move one block through the buffer data port. Caller buffers are only
+    // byte aligned in general, so unaligned ones go through a bounce copy.
+    static void PioBlock(uint8_t* data, bool write) {
+        uint32_t bounce[BlockSize / 4];
+        bool aligned = ((uintptr_t)data & 3) == 0;
+        uint32_t* words = aligned ? (uint32_t*)data : bounce;
+        if (write && !aligned) memcpy(bounce, data, BlockSize);
+        for (uint32_t i = 0; i < BlockSize / 4; i++) {
+            if (write) Write(REG_BUFFER_DATA, words[i]);
+            else words[i] = Read(REG_BUFFER_DATA);
+        }
+        if (!write && !aligned) memcpy(data, bounce, BlockSize);
+    }
+
+    // One command for `count` consecutive blocks: CMD17/CMD24 for a single
+    // block, else CMD18/CMD25 with the controller sending the closing CMD12
+    // itself (Auto CMD12; the BCM2711's EMMC2 wants it for multi-block reads).
+    // Cards program a multi-block write a flash page at a time; issued one
+    // block per command, each 512 bytes costs a full page program, which made
+    // writes (the first-boot grow above all) take tens of minutes.
+    static bool TransferBlocks(uint64_t lba, uint32_t count, uint8_t* data, bool write) {
+        uint32_t address = g_highCapacity ? (uint32_t)lba : (uint32_t)(lba * BlockSize);
+        bool multi = count > 1;
+        Write(REG_BLOCK_SIZE_COUNT, count << 16 | BlockSize);
+
+        uint32_t mode = write ? 0 : XM_READ;
+        if (multi) mode |= XM_BLOCK_COUNT_EN | XM_AUTO_CMD12 | XM_MULTI_BLOCK;
+        uint32_t command = write ? (multi ? 25 : 24) : (multi ? 18 : 17);
+
+        bool ok = SendCommand(command, address, Resp48, mode, true);
+        for (uint32_t i = 0; ok && i < count; i++) {
+            ok = WaitStatus(write ? INT_WRITE_READY : INT_READ_READY, DataTimeoutMs);
+            if (ok) PioBlock(data + (uint64_t)i * BlockSize, write);
+        }
+        ok = ok && WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs);
+
+        // A failed multi-block transfer leaves the card sending or receiving
+        // data; stop it so the next command finds it in the transfer state.
+        if (!ok && multi) SendCommand(12, 0, Resp48Busy);
+        return ok;
+    }
+
+    static bool TransferSectors(uint64_t lba, uint32_t count, uint8_t* data, bool write) {
+        if (lba + count > g_sectorCount) return false;
+        g_lock.Acquire();
+        bool ok = true;
+        while (ok && count > 0) {
+            uint32_t run = count > MaxBlocksPerCommand ? MaxBlocksPerCommand : count;
+            ok = TransferBlocks(lba, run, data, write);
+            lba += run;
+            data += (uint64_t)run * BlockSize;
+            count -= run;
+        }
+        g_lock.Release();
+        return ok;
+    }
+
+    static bool ReadSectors(void*, uint64_t lba, uint32_t count, void* buffer) {
+        return TransferSectors(lba, count, (uint8_t*)buffer, false);
+    }
+
+    static bool WriteSectors(void*, uint64_t lba, uint32_t count, const void* buffer) {
+        return TransferSectors(lba, count, (uint8_t*)buffer, true);
+    }
+
+    static bool InitializeController(uint64_t mmioPhys, const char* name) {
+        if (g_initialized) return false;
+
+        Memory::VMM::g_paging->MapMMIO(mmioPhys & ~0xFFFULL, Memory::HHDM(mmioPhys & ~0xFFFULL));
+        g_regs = (volatile uint8_t*)Memory::HHDM(mmioPhys);
+
+        if (!Reset(RESET_ALL)) {
+            KernelLogStream(ERROR, "SDHCI") << name << ": reset timed out";
+            return false;
+        }
+
+        g_version = (Read(REG_VERSION) >> 16) & 0xFF;
+        uint32_t caps = Read(REG_CAPABILITIES);
+        uint32_t baseMhz = g_version >= 2 ? (caps >> 8) & 0xFF : (caps >> 8) & 0x3F;
+        g_baseClockHz = baseMhz != 0 ? baseMhz * 1000000 : FallbackBaseClockHz;
+
+        uint32_t voltage = (caps & CAP_VOLT_330) ? PWR_330 : (caps & CAP_VOLT_300) ? PWR_300 : PWR_180;
+        Write(REG_HOST_CONTROL, (Read(REG_HOST_CONTROL) & ~0xFF00u) | voltage | PWR_ON);
+
+        Write(REG_INT_ENABLE, 0xFFFFFFFF);
+        Write(REG_INT_SIGNAL, 0);
+        Write(REG_CLOCK_CONTROL, (Read(REG_CLOCK_CONTROL) & ~0xFF0000u) | TIMEOUT_MAX);
+
+        if (!SetClock(400000)) {
+            KernelLogStream(ERROR, "SDHCI") << name << ": clock did not stabilize";
+            return false;
+        }
+
+        // The card needs 74 clocks after power-up before its first command.
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Timekeeping::GetMilliseconds() - start < 2) Arch::Pause();
+
+        KernelLogStream(INFO, "SDHCI") << name << ": version " << base::dec
+            << (uint64_t)(g_version + 1) << ".00, base clock " << (uint64_t)(g_baseClockHz / 1000000)
+            << " MHz" << (baseMhz == 0 ? " (assumed)" : "");
+
+        if (!InitializeCard()) {
+            KernelLogStream(INFO, "SDHCI") << name << ": no SD card";
+            return false;
+        }
+
+        g_initialized = true;
+        KernelLogStream(OK, "SDHCI") << name << ": " << (g_highCapacity ? "SDHC/SDXC" : "SDSC")
+            << " card, " << base::dec << (g_sectorCount * BlockSize / (1024 * 1024)) << " MiB";
+
+        BlockDevice dev = {};
+        dev.ReadSectors = ReadSectors;
+        dev.WriteSectors = WriteSectors;
+        dev.SectorCount = g_sectorCount;
+        dev.SectorSize = BlockSize;
+        dev.Kind = BLOCK_KIND_SD;
+        const char model[] = "SD card";
+        memcpy(dev.Model, model, sizeof(model));
+        RegisterBlockDevice(dev);
+        return true;
+    }
+
+    bool Probe(const Pci::PciDevice& dev) {
+        uint64_t bar = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
+        if (bar == 0) return false;
+        Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
+        return InitializeController(bar, "PCI controller");
+    }
+
+    bool ProbePlatform(uint64_t mmioPhys, const char* name) {
+        return InitializeController(mmioPhys, name);
+    }
+
+    bool IsInitialized() {
+        return g_initialized;
+    }
+
+}

kernel/src/Drivers/Storage/Sdhci.hpp

@@ -0,0 +1,22 @@
+/*
+    * Sdhci.hpp
+    * SD Host Controller (SDHCI) driver: SD cards as block devices
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+#include <Pci/Pci.hpp>
+
+namespace Drivers::Storage::Sdhci {
+
+    // A standard SD host controller on PCI (class 08h/05h).
+    bool Probe(const Pci::PciDevice& dev);
+
+    // One ACPI describes (the Raspberry Pi 4's EMMC2, BRCME88C; a generic
+    // PNP0D40 controller): its register block.
+    bool ProbePlatform(uint64_t mmioPhys, const char* name);
+
+    bool IsInitialized();
+
+}

kernel/src/Drivers/USB/Hub.cpp

@@ -0,0 +1,376 @@
+/*
+    * Hub.cpp
+    * USB hub class driver (USB 2.0 and SuperSpeed hubs)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Hub.hpp"
+#include "Xhci.hpp"
+#include "UsbDevice.hpp"
+#include <Arch/Cpu.hpp>
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
+#include <Timekeeping/Timer.hpp>
+#include <atomic>
+
+using namespace Kt;
+
+static void BusyWaitMs(uint64_t ms) {
+    if (Arch::InterruptsEnabled()) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Timekeeping::GetMilliseconds() - start < ms) {
+            Arch::Pause();
+        }
+    } else {
+        // The tick is not advancing: count ~1 us delays instead
+        for (uint64_t i = 0; i < ms * 1000; i++) {
+            Arch::IoDelay();
+        }
+    }
+}
+
+namespace Drivers::USB::Hub {
+
+    // -------------------------------------------------------------------------
+    // Hub class requests and port status (USB 2.0 11.24, USB 3.2 10.16)
+    // -------------------------------------------------------------------------
+
+    constexpr uint8_t REQTYPE_HUB_IN     = 0xA0;  // Dev-to-host, class, device
+    constexpr uint8_t REQTYPE_HUB_OUT    = 0x20;  // Host-to-dev, class, device
+    constexpr uint8_t REQTYPE_PORT_IN    = 0xA3;  // Dev-to-host, class, other (port)
+    constexpr uint8_t REQTYPE_PORT_OUT   = 0x23;  // Host-to-dev, class, other (port)
+
+    constexpr uint8_t REQ_GET_STATUS     = 0x00;
+    constexpr uint8_t REQ_CLEAR_FEATURE  = 0x01;
+    constexpr uint8_t REQ_SET_FEATURE    = 0x03;
+    constexpr uint8_t REQ_GET_DESCRIPTOR = 0x06;
+    constexpr uint8_t REQ_SET_HUB_DEPTH  = 0x0C;  // SuperSpeed hubs only
+
+    constexpr uint8_t DESC_HUB           = 0x29;
+    constexpr uint8_t DESC_SS_HUB        = 0x2A;
+
+    // Port feature selectors
+    constexpr uint16_t FEAT_PORT_RESET   = 4;
+    constexpr uint16_t FEAT_PORT_POWER   = 8;
+
+    // wPortStatus
+    constexpr uint16_t PS_CONNECTION     = (1 << 0);
+    constexpr uint16_t PS_ENABLE         = (1 << 1);
+    constexpr uint16_t PS_RESET          = (1 << 4);
+    constexpr uint16_t PS_LOW_SPEED      = (1 << 9);   // USB 2 hubs
+    constexpr uint16_t PS_HIGH_SPEED     = (1 << 10);  // USB 2 hubs
+
+    // wPortChange
+    constexpr uint16_t PC_CONNECTION     = (1 << 0);
+    constexpr uint16_t PC_RESET          = (1 << 4);
+
+    // The route string holds a 4-bit port number per tier, five tiers.
+    constexpr uint8_t MAX_TIERS          = 5;
+    constexpr uint8_t MAX_ROUTABLE_PORT  = 15;
+
+    // -------------------------------------------------------------------------
+    // State
+    // -------------------------------------------------------------------------
+
+    struct HubState {
+        bool              Active;
+        HubDescriptorInfo Info;
+    };
+
+    static HubState g_hubs[Xhci::MAX_SLOTS + 1] = {};
+
+    // Written from the event-ring drain, consumed by ProcessPendingWork.
+    static std::atomic<uint32_t> g_changedPorts[Xhci::MAX_SLOTS + 1] = {};  // bit n = port n
+    static std::atomic<bool>     g_rearm[Xhci::MAX_SLOTS + 1] = {};
+    static std::atomic<bool>     g_endpointFailed[Xhci::MAX_SLOTS + 1] = {};
+    static std::atomic<bool>     g_pending{false};
+
+    // -------------------------------------------------------------------------
+    // Requests
+    // -------------------------------------------------------------------------
+
+    static bool Succeeded(uint32_t cc) {
+        return cc == Xhci::CC_SUCCESS || cc == Xhci::CC_SHORT_PACKET;
+    }
+
+    static bool GetPortStatus(uint8_t hubSlot, uint8_t port, uint16_t& status, uint16_t& change) {
+        uint8_t buf[4] = {};
+        uint32_t cc = Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_IN, REQ_GET_STATUS,
+                                            0, port, sizeof(buf), buf, true);
+        if (!Succeeded(cc)) return false;
+        status = (uint16_t)(buf[0] | (buf[1] << 8));
+        change = (uint16_t)(buf[2] | (buf[3] << 8));
+        return true;
+    }
+
+    static uint32_t SetPortFeature(uint8_t hubSlot, uint8_t port, uint16_t feature) {
+        return Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_OUT, REQ_SET_FEATURE,
+                                     feature, port, 0, nullptr, false);
+    }
+
+    static uint32_t ClearPortFeature(uint8_t hubSlot, uint8_t port, uint16_t feature) {
+        return Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_OUT, REQ_CLEAR_FEATURE,
+                                     feature, port, 0, nullptr, false);
+    }
+
+    // Acknowledge every change bit in `change`. Bits 0-4 clear with
+    // C_PORT_CONNECTION..C_PORT_RESET (16-20); SuperSpeed hubs add
+    // C_BH_PORT_RESET, C_PORT_LINK_STATE and C_PORT_CONFIG_ERROR.
+    static void ClearPortChanges(uint8_t hubSlot, uint8_t port, uint16_t change, bool superSpeed) {
+        for (uint16_t bit = 0; bit <= 4; bit++) {
+            if (change & (1u << bit)) ClearPortFeature(hubSlot, port, (uint16_t)(16 + bit));
+        }
+        if (superSpeed) {
+            if (change & (1u << 5)) ClearPortFeature(hubSlot, port, 29);
+            if (change & (1u << 6)) ClearPortFeature(hubSlot, port, 25);
+            if (change & (1u << 7)) ClearPortFeature(hubSlot, port, 26);
+        }
+    }
+
+    // Acknowledge hub-wide changes (local power, over-current): until they
+    // are cleared the hub keeps reporting bit 0 of its change bitmap.
+    static void ClearHubChanges(uint8_t hubSlot) {
+        uint8_t buf[4] = {};
+        uint32_t cc = Xhci::ControlTransfer(hubSlot, REQTYPE_HUB_IN, REQ_GET_STATUS,
+                                            0, 0, sizeof(buf), buf, true);
+        if (!Succeeded(cc)) return;
+        uint16_t change = (uint16_t)(buf[2] | (buf[3] << 8));
+        for (uint16_t bit = 0; bit <= 1; bit++) {  // C_HUB_LOCAL_POWER, C_HUB_OVER_CURRENT
+            if (change & (1u << bit)) {
+                Xhci::ControlTransfer(hubSlot, REQTYPE_HUB_OUT, REQ_CLEAR_FEATURE,
+                                      bit, 0, 0, nullptr, false);
+            }
+        }
+    }
+
+    bool ReadDescriptor(uint8_t slotId, uint32_t speed, HubDescriptorInfo& out) {
+        out = {};
+        out.SuperSpeed = speed >= Xhci::SPEED_SUPER;
+        uint8_t type = out.SuperSpeed ? DESC_SS_HUB : DESC_HUB;
+
+        uint8_t buf[16] = {};
+        uint32_t cc = Xhci::ControlTransfer(slotId, REQTYPE_HUB_IN, REQ_GET_DESCRIPTOR,
+                                            (uint16_t)(type << 8), 0, sizeof(buf), buf, true);
+        if (!Succeeded(cc) || buf[0] < 7 || buf[1] != type || buf[2] == 0) {
+            KernelLogStream(ERROR, "USB/Hub") << "Slot " << base::dec << (uint64_t)slotId
+                << ": hub descriptor unreadable, cc=" << (uint64_t)cc;
+            return false;
+        }
+
+        uint16_t characteristics = (uint16_t)(buf[3] | (buf[4] << 8));
+        out.Ports = buf[2];
+        out.ThinkTime = out.SuperSpeed ? 0 : (uint8_t)((characteristics >> 5) & 3);
+        out.PowerOnMs = (uint16_t)(buf[5] * 2);
+        return true;
+    }
+
+    // -------------------------------------------------------------------------
+    // Ports
+    // -------------------------------------------------------------------------
+
+    static uint8_t FindChild(uint8_t hubSlot, uint8_t port) {
+        for (uint8_t s = 1; s <= Xhci::MAX_SLOTS; s++) {
+            auto* dev = Xhci::GetDevice(s);
+            if (dev->Active && dev->ParentSlot == hubSlot && dev->ParentPort == port) return s;
+        }
+        return 0;
+    }
+
+    // Reset a connected port and report the speed the device came up at.
+    static bool ResetPort(uint8_t hubSlot, uint8_t port, bool superSpeed, uint32_t& speed) {
+        if (SetPortFeature(hubSlot, port, FEAT_PORT_RESET) != Xhci::CC_SUCCESS) return false;
+
+        uint16_t status = 0, change = 0;
+        bool done = false;
+        for (int attempt = 0; attempt < 50; attempt++) {  // ~500 ms
+            BusyWaitMs(10);
+            if (!GetPortStatus(hubSlot, port, status, change)) return false;
+            if ((change & PC_RESET) && !(status & PS_RESET)) { done = true; break; }
+            if (!(status & PS_CONNECTION)) return false;
+        }
+        ClearPortChanges(hubSlot, port, change, superSpeed);
+        if (!done || !(status & PS_ENABLE)) return false;
+
+        if (superSpeed)                   speed = Xhci::SPEED_SUPER;
+        else if (status & PS_LOW_SPEED)   speed = Xhci::SPEED_LOW;
+        else if (status & PS_HIGH_SPEED)  speed = Xhci::SPEED_HIGH;
+        else                              speed = Xhci::SPEED_FULL;
+        return true;
+    }
+
+    // Bring the port's state in line with what is attached to it: remove a
+    // device that left (or was replaced), enumerate one that arrived.
+    static void HandlePort(uint8_t hubSlot, uint8_t port, bool settled) {
+        const HubDescriptorInfo& info = g_hubs[hubSlot].Info;
+
+        uint16_t status = 0, change = 0;
+        if (!GetPortStatus(hubSlot, port, status, change)) return;
+        if (change != 0) ClearPortChanges(hubSlot, port, change, info.SuperSpeed);
+
+        bool connected = (status & PS_CONNECTION) != 0;
+        uint8_t child = FindChild(hubSlot, port);
+        if (child != 0 && (!connected || (change & PC_CONNECTION))) {
+            KernelLogStream(INFO, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
+                << " port " << (uint64_t)port << ": device disconnected";
+            Xhci::RemoveDevice(child);
+            child = 0;
+        }
+        if (!connected || child != 0) return;
+
+        if (!settled) {
+            // Connect debounce (USB 2.0 7.1.7.3): let contacts settle first.
+            BusyWaitMs(100);
+            if (!GetPortStatus(hubSlot, port, status, change)) return;
+            if (!(status & PS_CONNECTION)) return;
+        }
+
+        auto* hub = Xhci::GetDevice(hubSlot);
+        if (hub->Depth + 1 > MAX_TIERS || port > MAX_ROUTABLE_PORT) {
+            KernelLogStream(WARNING, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
+                << " port " << (uint64_t)port << ": too deep to address, ignored";
+            return;
+        }
+
+        uint32_t speed = 0;
+        if (!ResetPort(hubSlot, port, info.SuperSpeed, speed)) {
+            KernelLogStream(WARNING, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
+                << " port " << (uint64_t)port << ": reset failed";
+            return;
+        }
+        BusyWaitMs(10);  // reset recovery (USB 2.0 7.1.7.5)
+
+        UsbDevice::DeviceLocation location = {};
+        location.RootPort    = hub->PortId;
+        location.ParentSlot  = hubSlot;
+        location.ParentPort  = port;
+        location.Depth       = (uint8_t)(hub->Depth + 1);
+        location.RouteString = hub->RouteString | ((uint32_t)port << (4 * hub->Depth));
+        if (hub->Speed == Xhci::SPEED_HIGH &&
+            (speed == Xhci::SPEED_LOW || speed == Xhci::SPEED_FULL)) {
+            // This hub's transaction translator turns the high-speed
+            // traffic into low/full-speed transactions. Single-TT mode:
+            // the configuration's default alternate setting.
+            location.TtHubSlot = hubSlot;
+            location.TtPort    = port;
+            location.TtMulti   = false;
+        } else {
+            location.TtHubSlot = hub->TtHubSlot;
+            location.TtPort    = hub->TtPort;
+            location.TtMulti   = hub->TtMulti;
+        }
+
+        UsbDevice::EnumerateDevice(location, speed);
+    }
+
+    // -------------------------------------------------------------------------
+    // Public API
+    // -------------------------------------------------------------------------
+
+    void RegisterHub(uint8_t slotId, const HubDescriptorInfo& info) {
+        auto* dev = Xhci::GetDevice(slotId);
+        if (dev == nullptr || !dev->Active) return;
+
+        g_hubs[slotId].Info = info;
+        g_hubs[slotId].Active = true;
+        g_changedPorts[slotId].store(0, std::memory_order_relaxed);
+        g_rearm[slotId].store(false, std::memory_order_relaxed);
+        g_endpointFailed[slotId].store(false, std::memory_order_relaxed);
+
+        uint8_t ports = info.Ports;
+        if (ports > 31) ports = 31;  // status-change bitmap tracked in 32 bits
+
+        KernelLogStream(OK, "USB/Hub") << "Slot " << base::dec << (uint64_t)slotId << ": "
+            << (info.SuperSpeed ? "SuperSpeed" : "USB 2.0") << " hub, "
+            << (uint64_t)info.Ports << " ports, tier " << (uint64_t)(dev->Depth + 1);
+
+        if (info.SuperSpeed) {
+            Xhci::ControlTransfer(slotId, REQTYPE_HUB_OUT, REQ_SET_HUB_DEPTH,
+                                  dev->Depth, 0, 0, nullptr, false);
+        }
+
+        for (uint8_t port = 1; port <= ports; port++) {
+            SetPortFeature(slotId, port, FEAT_PORT_POWER);
+        }
+        // Power-on-to-power-good, then the connect debounce interval.
+        uint64_t wait = info.PowerOnMs < 100 ? 100 : info.PowerOnMs;
+        BusyWaitMs(wait + 100);
+
+        for (uint8_t port = 1; port <= ports && g_hubs[slotId].Active; port++) {
+            HandlePort(slotId, port, true);
+        }
+
+        if (g_hubs[slotId].Active) Xhci::QueueInterruptTransfer(slotId);
+    }
+
+    void UnregisterHub(uint8_t slotId) {
+        if (slotId == 0 || slotId > Xhci::MAX_SLOTS) return;
+        g_hubs[slotId] = {};
+        g_changedPorts[slotId].store(0, std::memory_order_relaxed);
+        g_rearm[slotId].store(false, std::memory_order_relaxed);
+        g_endpointFailed[slotId].store(false, std::memory_order_relaxed);
+    }
+
+    void OnStatusChange(uint8_t slotId, const uint8_t* data, uint16_t length,
+                        uint32_t completionCode) {
+        if (slotId == 0 || slotId > Xhci::MAX_SLOTS) return;
+
+        if (Succeeded(completionCode)) {
+            uint32_t bits = 0;
+            for (uint16_t i = 0; i < length && i < 4; i++) {
+                bits |= (uint32_t)data[i] << (8 * i);
+            }
+            g_changedPorts[slotId].fetch_or(bits, std::memory_order_relaxed);
+            g_rearm[slotId].store(true, std::memory_order_relaxed);
+        } else {
+            // The endpoint halted; it needs a reset before it reports again,
+            // and changes may have been missed meanwhile.
+            g_endpointFailed[slotId].store(true, std::memory_order_relaxed);
+        }
+        g_pending.store(true, std::memory_order_release);
+    }
+
+    void RequestPortCheck(uint8_t hubSlot, uint8_t port) {
+        if (hubSlot == 0 || hubSlot > Xhci::MAX_SLOTS || port == 0 || port > 31) return;
+        g_changedPorts[hubSlot].fetch_or(1u << port, std::memory_order_relaxed);
+        g_pending.store(true, std::memory_order_release);
+    }
+
+    bool ResetChildPort(uint8_t hubSlot, uint8_t port, uint32_t& speed) {
+        if (hubSlot == 0 || hubSlot > Xhci::MAX_SLOTS || !g_hubs[hubSlot].Active) return false;
+        return ResetPort(hubSlot, port, g_hubs[hubSlot].Info.SuperSpeed, speed);
+    }
+
+    bool HasPendingWork() {
+        return g_pending.load(std::memory_order_acquire);
+    }
+
+    void ProcessPendingWork() {
+        if (!g_pending.exchange(false, std::memory_order_acq_rel)) return;
+
+        for (uint8_t slot = 1; slot <= Xhci::MAX_SLOTS; slot++) {
+            if (!g_hubs[slot].Active) continue;
+
+            bool failed = g_endpointFailed[slot].exchange(false, std::memory_order_acq_rel);
+            bool rearm = g_rearm[slot].exchange(false, std::memory_order_acq_rel);
+            uint32_t changed = g_changedPorts[slot].exchange(0, std::memory_order_acq_rel);
+            if (!failed && !rearm && changed == 0) continue;
+
+            uint8_t ports = g_hubs[slot].Info.Ports;
+            if (ports > 31) ports = 31;
+
+            if (failed) {
+                // Reset Endpoint re-arms the endpoint itself.
+                Xhci::ResetInterruptEndpoint(slot);
+                changed = ((ports == 31) ? 0xFFFFFFFFu : ((1u << (ports + 1)) - 1)) & ~1u;
+            }
+
+            if (changed & 1u) ClearHubChanges(slot);
+            for (uint8_t port = 1; port <= ports && g_hubs[slot].Active; port++) {
+                if (changed & (1u << port)) HandlePort(slot, port, false);
+            }
+
+            if (rearm && !failed && g_hubs[slot].Active) Xhci::QueueInterruptTransfer(slot);
+        }
+    }
+
+}

kernel/src/Drivers/USB/Hub.hpp

@@ -0,0 +1,61 @@
+/*
+    * Hub.hpp
+    * USB hub class driver (USB 2.0 and SuperSpeed hubs)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// Devices behind a hub get their own xHCI slots, addressed by route string
+// (Xhci::UsbDeviceInfo). The hub driver powers and resets the hub's ports
+// and enumerates what it finds; the hub's interrupt endpoint reports port
+// changes afterwards, handled from the xHCI deferred-work pass.
+//
+// The Raspberry Pi 4/400 needs this for any USB 2 device: the VL805's only
+// USB 2 root port carries an on-board VIA hub that every USB 2 line (the
+// Pi 400's built-in keyboard included) hangs off.
+
+namespace Drivers::USB::Hub {
+
+    struct HubDescriptorInfo {
+        uint8_t  Ports;
+        uint8_t  ThinkTime;     // TT think time code, wHubCharacteristics[6:5]
+        uint16_t PowerOnMs;     // bPwrOn2PwrGood, in ms
+        bool     SuperSpeed;
+    };
+
+    // Read the hub class descriptor of a configured hub. `speed` is the
+    // hub's xHCI port speed (SuperSpeed hubs have their own descriptor).
+    bool ReadDescriptor(uint8_t slotId, uint32_t speed, HubDescriptorInfo& out);
+
+    // Take over a hub whose slot is configured as a hub with its interrupt
+    // endpoint: power its ports, enumerate the devices on them, then arm the
+    // status-change endpoint. Process context.
+    void RegisterHub(uint8_t slotId, const HubDescriptorInfo& info);
+
+    // Forget a hub (its children are removed by Xhci::RemoveDevice).
+    void UnregisterHub(uint8_t slotId);
+
+    // A completion on the hub's status-change endpoint. Called from the
+    // event ring drain: records the change and nothing else.
+    void OnStatusChange(uint8_t slotId, const uint8_t* data, uint16_t length,
+                        uint32_t completionCode);
+
+    // Look at a port again on the next ProcessPendingWork as if it had
+    // reported a change: a device removed from it is reset and enumerated
+    // afresh, as if replugged.
+    void RequestPortCheck(uint8_t hubSlot, uint8_t port);
+
+    // Reset a hub port again (an enumeration retry) and report the speed the
+    // device came up at. False if the hub is gone or the reset failed.
+    bool ResetChildPort(uint8_t hubSlot, uint8_t port, uint32_t& speed);
+
+    // Port changes are waiting for ProcessPendingWork.
+    bool HasPendingWork();
+
+    // Handle recorded port changes and re-arm the endpoints. Runs inside the
+    // xHCI deferred-work pass (process/idle context, single runner).
+    void ProcessPendingWork();
+
+}

kernel/src/Drivers/USB/MassStorage.cpp

@@ -51,6 +51,16 @@ namespace Drivers::USB::MassStorage {
 
     constexpr uint8_t SERVICE_READ_CAPACITY16 = 0x10;
 
+    constexpr uint8_t SENSE_NOT_READY        = 0x02;
+    constexpr uint8_t SENSE_UNIT_ATTENTION   = 0x06;
+    constexpr uint8_t ASC_MEDIUM_NOT_PRESENT = 0x3A;
+
+    // TEST UNIT READY attempts, 100 ms apart: any failure gets a few, a unit
+    // that says it is coming up (see TestUnitReady) gets about five seconds.
+    constexpr int TUR_TRIES         = 5;
+    constexpr int TUR_SETTLE_TRIES  = 50;
+    constexpr int TUR_NO_MEDIUM_TRIES = 3;
+
     constexpr uint32_t MAX_BULK_TRANSFER_BYTES = 0x10000;
     constexpr int MAX_MSC_DEVICES = 16;
 
@@ -275,6 +285,15 @@ namespace Drivers::USB::MassStorage {
         return maxLun;
     }
 
+    // A failed stage leaves its endpoint halted, or after a timeout its TRB
+    // still queued; either way the next transfer would fail too.
+    static uint32_t BulkStage(Device& dev, bool dirIn, void* data, uint64_t dataPhys,
+                              uint32_t length, uint32_t* actual) {
+        uint32_t cc = Xhci::BulkTransfer(dev.SlotId, dirIn, data, dataPhys, length, actual);
+        if (!IsGoodCompletion(cc)) Xhci::RecoverBulkEndpoint(dev.SlotId, dirIn);
+        return cc;
+    }
+
     static bool ScsiCommand(Device& dev, const uint8_t* cdb, uint8_t cdbLen,
                             void* data, uint64_t dataPhys, uint32_t dataLen,
                             bool dataIn) {
@@ -295,24 +314,34 @@ namespace Drivers::USB::MassStorage {
         memcpy(cbw->CommandBlock, cdb, cdbLen);
 
         uint32_t actual = 0;
-        uint32_t cc = Xhci::BulkTransfer(dev.SlotId, false, dev.CbwBuf, dev.CbwPhys,
+        uint32_t cc = BulkStage(dev, false, dev.CbwBuf, dev.CbwPhys,
                                 sizeof(CommandBlockWrapper), &actual);
         if (!IsGoodCompletion(cc) || actual != sizeof(CommandBlockWrapper)) {
             ReleaseTransport();
             return false;
         }
 
+        // A device that fails the command, or has less data than asked for,
+        // may STALL the data stage; the status still follows in the CSW once
+        // the halt is cleared (BOT 6.7.2, 6.7.3). Card readers do this for
+        // commands on a slot that is not ready.
         if (dataLen > 0) {
-            cc = Xhci::BulkTransfer(dev.SlotId, dataIn, data, dataPhys, dataLen, &actual);
-            if (!IsGoodCompletion(cc)) {
+            cc = BulkStage(dev, dataIn, data, dataPhys, dataLen, &actual);
+            if (!IsGoodCompletion(cc) && cc != Xhci::CC_STALL_ERROR) {
                 ReleaseTransport();
                 return false;
             }
         }
 
+        // A STALL on the CSW itself is cleared and the CSW read again once
+        // (BOT 5.3.3, figure 2).
         memset(dev.CswBuf, 0, 0x1000);
-        cc = Xhci::BulkTransfer(dev.SlotId, true, dev.CswBuf, dev.CswPhys,
+        cc = BulkStage(dev, true, dev.CswBuf, dev.CswPhys,
                        sizeof(CommandStatusWrapper), &actual);
+        if (cc == Xhci::CC_STALL_ERROR) {
+            cc = BulkStage(dev, true, dev.CswBuf, dev.CswPhys,
+                           sizeof(CommandStatusWrapper), &actual);
+        }
         if (!IsGoodCompletion(cc) || actual < sizeof(CommandStatusWrapper)) {
             ReleaseTransport();
             return false;
@@ -335,35 +364,66 @@ namespace Drivers::USB::MassStorage {
     // SCSI commands
     // -------------------------------------------------------------------------
 
-    static bool RequestSense(Device& dev) {
+    struct Sense {
+        uint8_t Key;
+        uint8_t Asc;
+        uint8_t Ascq;
+    };
+
+    static bool RequestSense(Device& dev, Sense& out) {
         uint8_t cdb[6] = {};
         cdb[0] = SCSI_REQUEST_SENSE;
         cdb[4] = 18;
 
         memset(dev.ScratchBuf, 0, 18);
-        bool ok = ScsiCommand(dev, cdb, 6, dev.ScratchBuf, dev.ScratchPhys, 18, true);
-        if (ok) {
-            uint8_t senseKey = dev.ScratchBuf[2] & 0x0F;
-            uint8_t asc = dev.ScratchBuf[12];
-            uint8_t ascq = dev.ScratchBuf[13];
-            KernelLogStream(INFO, "USB-MSC") << "REQUEST SENSE: key="
-                << (uint64_t)senseKey << " asc=" << base::hex << (uint64_t)asc
-                << " ascq=" << (uint64_t)ascq << base::dec;
-        }
-        return ok;
+        if (!ScsiCommand(dev, cdb, 6, dev.ScratchBuf, dev.ScratchPhys, 18, true)) {
+            return false;
         }
 
+        out.Key = dev.ScratchBuf[2] & 0x0F;
+        out.Asc = dev.ScratchBuf[12];
+        out.Ascq = dev.ScratchBuf[13];
+        return true;
+    }
+
+    // Right after a reset a unit reports UNIT ATTENTION (power on, medium
+    // changed), and a card reader then reports NOT READY while it brings the
+    // card up, which can take well over a second. Keep asking while the sense
+    // says the unit is on its way; an empty slot (MEDIUM NOT PRESENT) is final.
     static bool TestUnitReady(Device& dev) {
         uint8_t cdb[6] = {};
         cdb[0] = SCSI_TEST_UNIT_READY;
 
-        for (int i = 0; i < 5; i++) {
+        int limit = TUR_TRIES;
+        int noMedium = 0;
+        Sense last = {0xFF, 0xFF, 0xFF};
+
+        for (int i = 0; ; i++) {
             if (ScsiCommand(dev, cdb, 6, nullptr, 0, 0, false)) return true;
-            RequestSense(dev);
-            PollWaitMs(50);
+
+            Sense sense = {};
+            if (RequestSense(dev, sense)) {
+                if (sense.Key != last.Key || sense.Asc != last.Asc || sense.Ascq != last.Ascq) {
+                    KernelLogStream(INFO, "USB-MSC") << "LUN " << (uint64_t)dev.Lun
+                        << " REQUEST SENSE: key=" << (uint64_t)sense.Key
+                        << " asc=" << base::hex << (uint64_t)sense.Asc
+                        << " ascq=" << (uint64_t)sense.Ascq << base::dec;
+                    last = sense;
                 }
 
-        return false;
+                if (sense.Key == SENSE_NOT_READY && sense.Asc == ASC_MEDIUM_NOT_PRESENT) {
+                    if (++noMedium >= TUR_NO_MEDIUM_TRIES) return false;
+                } else {
+                    noMedium = 0;
+                    if (sense.Key == SENSE_NOT_READY || sense.Key == SENSE_UNIT_ATTENTION) {
+                        limit = TUR_SETTLE_TRIES;
+                    }
+                }
+            }
+
+            if (i + 1 >= limit) return false;
+            PollWaitMs(100);
+        }
     }
 
     static bool Inquiry(Device& dev) {
@@ -621,7 +681,14 @@ namespace Drivers::USB::MassStorage {
                 continue;
             }
 
-            if (!ReadCapacity(dev)) {
+            // Some readers fail the first READ CAPACITY after the unit has
+            // just become ready.
+            bool haveCapacity = false;
+            for (int attempt = 0; attempt < 3 && !haveCapacity; attempt++) {
+                if (attempt > 0) PollWaitMs(100);
+                haveCapacity = ReadCapacity(dev);
+            }
+            if (!haveCapacity) {
                 KernelLogStream(WARNING, "USB-MSC") << "READ CAPACITY failed for LUN "
                     << (uint64_t)lun;
                 dev.Active = false;

kernel/src/Drivers/USB/UsbDevice.cpp

@@ -10,7 +10,8 @@
 #include "HidKeyboard.hpp"
 #include "HidMouse.hpp"
 #include "MassStorage.hpp"
-#include "Bluetooth/Bluetooth.hpp"
+#include "Hub.hpp"
+#include <Drivers/Bluetooth/Bluetooth.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 #include <Memory/HHDM.hpp>
@@ -65,24 +66,28 @@ namespace Drivers::USB::UsbDevice {
         return speed;  // Same encoding
     }
 
-    // Convert USB endpoint bInterval to xHCI Endpoint Context Interval value.
-    // HS/SS: bInterval is already in 2^(n-1) * 125µs encoding — use directly.
-    // FS/LS: bInterval is in milliseconds (frames) — convert via fls(bInterval * 8).
+    // Convert USB endpoint bInterval to xHCI Endpoint Context Interval value
+    // (a period of 2^Interval * 125 us), as Linux's xhci_get_endpoint_interval.
+    // HS/SS: bInterval encodes a period of 2^(bInterval-1) microframes.
+    // FS/LS: bInterval is in milliseconds (frames); round down to a power of
+    // two microframes, between 1 ms and 128 ms.
     static uint32_t ConvertInterval(uint32_t speed, uint8_t bInterval) {
-        if (bInterval == 0) return 0;
-
         if (speed == Xhci::SPEED_HIGH || speed == Xhci::SPEED_SUPER) {
-            return bInterval;
+            uint32_t b = bInterval;
+            if (b < 1) b = 1;
+            if (b > 16) b = 16;
+            return b - 1;
         }
 
-        // FS/LS: bInterval ms → microframes, then find highest set bit position
+        if (bInterval == 0) return 3;
         uint32_t microframes = (uint32_t)bInterval * 8;
         uint32_t interval = 0;
-        while (microframes > 0) {
+        while (microframes > 1) {
             interval++;
             microframes >>= 1;
         }
-        if (interval > 15) interval = 15;
+        if (interval < 3) interval = 3;
+        if (interval > 10) interval = 10;
         return interval;
     }
 
@@ -101,13 +106,36 @@ namespace Drivers::USB::UsbDevice {
     // EnumerateDevice
     // ---------------------------------------------------------------------------
 
-    uint8_t EnumerateDevice(uint8_t portId, uint32_t speed) {
-        KernelLogStream(INFO, "USB") << "Enumerating device on port "
-            << (uint64_t)portId << " speed=" << SpeedToString(speed);
+    static bool EnumerateInSlot(uint8_t slotId, const DeviceLocation& location, uint32_t speed,
+                                bool& isHub, Hub::HubDescriptorInfo& hubInfo, bool& retryable);
 
-        // -----------------------------------------------------------------
+    // Attempts at bringing a device up, each after a fresh port reset (as
+    // Linux's hub_port_init): a device can fail its first requests with a
+    // transaction error while it is still settling after the reset. Card
+    // readers do, and USB devices the firmware used to boot from.
+    constexpr int ENUM_TRIES = 3;
+
+    uint8_t EnumerateDevice(uint8_t portId, uint32_t speed) {
+        DeviceLocation location = {};
+        location.RootPort = portId;
+        return EnumerateDevice(location, speed);
+    }
+
+    uint8_t EnumerateDevice(const DeviceLocation& location, uint32_t speed) {
+        if (location.ParentSlot == 0) {
+            KernelLogStream(INFO, "USB") << "Enumerating device on port "
+                << (uint64_t)location.RootPort << " speed=" << SpeedToString(speed);
+        } else {
+            KernelLogStream(INFO, "USB") << "Enumerating device on hub slot "
+                << (uint64_t)location.ParentSlot << " port " << (uint64_t)location.ParentPort
+                << " (route " << base::hex << (uint64_t)location.RouteString << base::dec
+                << ", root port " << (uint64_t)location.RootPort << ") speed=" << SpeedToString(speed);
+        }
+
+        for (int attempt = 1; ; attempt++) {
+            // -------------------------------------------------------------
             // Step 1: Enable Slot
-        // -----------------------------------------------------------------
+            // -------------------------------------------------------------
             Xhci::TRB enableSlotTrb = {};
             enableSlotTrb.Control = (Xhci::TRB_ENABLE_SLOT << Xhci::TRB_TYPE_SHIFT);
 
@@ -125,6 +153,54 @@ namespace Drivers::USB::UsbDevice {
 
             KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId << " enabled";
 
+            bool isHub = false;
+            bool retryable = false;
+            Hub::HubDescriptorInfo hubInfo = {};
+            if (EnumerateInSlot(slotId, location, speed, isHub, hubInfo, retryable)) {
+                // A hub enumerates what is behind it now that its own slot is
+                // published (its children name it as their parent).
+                if (isHub) Hub::RegisterHub(slotId, hubInfo);
+                return slotId;
+            }
+
+            // Give the slot back, or every failed attempt (a flaky device
+            // re-reported by its hub) would leak one until none are left.
+            Xhci::TRB disableTrb = {};
+            disableTrb.Control = (Xhci::TRB_DISABLE_SLOT << Xhci::TRB_TYPE_SHIFT)
+                               | ((uint32_t)slotId << 24);
+            Xhci::SendCommand(disableTrb);
+            Xhci::g_dcbaa[slotId] = 0;
+            *Xhci::GetDevice(slotId) = {};
+
+            if (!retryable || attempt >= ENUM_TRIES) return 0;
+
+            // Back the device off to its default state, giving it longer to
+            // settle each time.
+            BusyWaitMs(100 * (uint64_t)attempt);
+            bool reset = location.ParentSlot == 0
+                ? Xhci::ResetRootPort(location.RootPort, speed)
+                : Hub::ResetChildPort(location.ParentSlot, location.ParentPort, speed);
+            if (!reset) {
+                KernelLogStream(WARNING, "USB") << "Port reset for retry failed, giving up";
+                return 0;
+            }
+            BusyWaitMs(10 * (uint64_t)(attempt + 1));  // reset recovery, and then some
+
+            KernelLogStream(INFO, "USB") << "Retrying enumeration (attempt "
+                << (uint64_t)(attempt + 1) << " of " << (uint64_t)ENUM_TRIES
+                << ") speed=" << SpeedToString(speed);
+        }
+    }
+
+    static bool EnumerateInSlot(uint8_t slotId, const DeviceLocation& location, uint32_t speed,
+                                bool& isHub, Hub::HubDescriptorInfo& hubInfo, bool& retryable) {
+        uint8_t portId = location.RootPort;
+        uint32_t cc = 0;
+
+        // Until the device is configured nothing has claimed it, so a failure
+        // up to then can be retried from a port reset.
+        retryable = true;
+
         // -----------------------------------------------------------------
         // Step 2: Allocate device output context and set DCBAA entry
         // -----------------------------------------------------------------
@@ -133,9 +209,16 @@ namespace Drivers::USB::UsbDevice {
         dev->Active = true;
         dev->PortId = portId;
         dev->Speed  = speed;
+        dev->ParentSlot  = location.ParentSlot;
+        dev->ParentPort  = location.ParentPort;
+        dev->Depth       = location.Depth;
+        dev->RouteString = location.RouteString;
+        dev->TtHubSlot   = location.TtHubSlot;
+        dev->TtPort      = location.TtPort;
+        dev->TtMulti     = location.TtMulti;
 
         // Allocate a zeroed page for the output DeviceContext
-        auto* outputCtx = (Xhci::DeviceContext*)Memory::g_pfa->AllocateZeroed();
+        auto* outputCtx = (Xhci::DeviceContext*)Xhci::AllocateDmaPage();
         dev->OutputContext     = outputCtx;
         dev->OutputContextPhys = Memory::SubHHDM(outputCtx);
 
@@ -145,7 +228,7 @@ namespace Drivers::USB::UsbDevice {
         // -----------------------------------------------------------------
         // Step 3: Build Input Context for Address Device command
         // -----------------------------------------------------------------
-        auto* inputCtx = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
+        auto* inputCtx = (Xhci::InputContext*)Xhci::AllocateDmaPage();
 
         // Input Control Context: add Slot Context (bit 0) and EP0 (bit 1)
         inputCtx->ICC.AddFlags = 0x3;
@@ -153,12 +236,18 @@ namespace Drivers::USB::UsbDevice {
         // Slot Context
         uint32_t speedVal    = SpeedToSlotContextValue(speed);
         uint32_t ctxEntries  = 1;  // Context Entries = 1 (Slot + EP0 only)
-        inputCtx->Slot.Field0 = (ctxEntries << 27) | (speedVal << 20);
+        inputCtx->Slot.Field0 = (ctxEntries << 27) | (speedVal << 20)
+                              | (location.RouteString & 0xFFFFF)
+                              | (location.TtMulti ? Xhci::SLOT_MTT : 0);
         inputCtx->Slot.Field1 = ((uint32_t)portId << 16);  // Root Hub Port Number
+        // Low/full-speed behind a high-speed hub: the hub slot and port
+        // whose transaction translator the controller addresses.
+        inputCtx->Slot.Field2 = (uint32_t)location.TtHubSlot
+                              | ((uint32_t)location.TtPort << 8);
 
         // EP0 Context
         // Allocate EP0 transfer ring
-        auto* ep0Ring = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+        auto* ep0Ring = (Xhci::TRB*)Xhci::AllocateDmaPage();
         dev->EP0Ring        = ep0Ring;
         dev->EP0RingPhys    = Memory::SubHHDM(ep0Ring);
         dev->EP0RingEnqueue = 0;
@@ -198,8 +287,7 @@ namespace Drivers::USB::UsbDevice {
         if (cc != Xhci::CC_SUCCESS) {
             KernelLogStream(ERROR, "USB") << "Address Device (BSR=1) failed, slot="
                 << (uint64_t)slotId << " cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId << " initialized (BSR=1)";
@@ -213,11 +301,15 @@ namespace Drivers::USB::UsbDevice {
                                    partialDesc, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(8-byte) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
-        uint8_t bMaxPacketSize0 = partialDesc[7];
+        // USB 3 devices give bMaxPacketSize0 as an exponent (9: 512 bytes).
+        uint16_t bMaxPacketSize0 = partialDesc[7];
+        if (speed >= Xhci::SPEED_SUPER) {
+            bMaxPacketSize0 = (bMaxPacketSize0 >= 5 && bMaxPacketSize0 <= 15)
+                ? (uint16_t)(1u << bMaxPacketSize0) : maxPacket;
+        }
         if (bMaxPacketSize0 == 0) bMaxPacketSize0 = maxPacket; // fallback
 
         KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId
@@ -227,7 +319,7 @@ namespace Drivers::USB::UsbDevice {
         // Step 4c: Evaluate Context — update EP0 max packet size if needed
         // -----------------------------------------------------------------
         if (bMaxPacketSize0 != maxPacket) {
-            auto* evalCtx = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
+            auto* evalCtx = (Xhci::InputContext*)Xhci::AllocateDmaPage();
 
             // Only updating EP0 — set AddFlags bit 1 (EP0), no slot context needed
             evalCtx->ICC.AddFlags = (1 << 1);
@@ -280,8 +372,7 @@ namespace Drivers::USB::UsbDevice {
         if (cc != Xhci::CC_SUCCESS) {
             KernelLogStream(ERROR, "USB") << "Address Device failed, slot="
                 << (uint64_t)slotId << " cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         // Set-address recovery time (USB spec requires >= 2ms, use 10ms for safety)
@@ -298,8 +389,7 @@ namespace Drivers::USB::UsbDevice {
                                    &devDesc, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Device) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         dev->VendorId    = devDesc.idVendor;
@@ -319,8 +409,7 @@ namespace Drivers::USB::UsbDevice {
                                    &cfgHdr, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Config header) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         uint8_t cfgBuf[256] = {};
@@ -332,8 +421,7 @@ namespace Drivers::USB::UsbDevice {
                                    cfgBuf, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Config full) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         // -----------------------------------------------------------------
@@ -341,6 +429,7 @@ namespace Drivers::USB::UsbDevice {
         // -----------------------------------------------------------------
         uint16_t offset = 0;
         bool foundHid = false;
+        bool currentHub = false;
         bool foundBt = false;
         bool foundMsc = false;
         bool currentMsc = false;
@@ -358,9 +447,23 @@ namespace Drivers::USB::UsbDevice {
                 auto* iface = (InterfaceDescriptor*)&cfgBuf[offset];
                 // Reset at each new interface boundary
                 foundHid = false;
+                currentHub = false;
                 foundBt = false;
                 currentMsc = false;
 
+                // Hub (class 9): its status-change interrupt endpoint. Only
+                // alternate setting 0 (single-TT) is used.
+                if (!foundEp && !isHub &&
+                    iface->bInterfaceClass == CLASS_HUB &&
+                    iface->bAlternateSetting == 0) {
+                    dev->InterfaceClass    = iface->bInterfaceClass;
+                    dev->InterfaceSubClass = iface->bInterfaceSubClass;
+                    dev->InterfaceProtocol = iface->bInterfaceProtocol;
+                    dev->InterfaceNumber   = iface->bInterfaceNumber;
+                    currentHub = true;
+                    isHub = true;
+                }
+
                 if (!foundEp &&
                     iface->bInterfaceClass == CLASS_HID &&
                     iface->bInterfaceSubClass == SUBCLASS_BOOT) {
@@ -408,8 +511,8 @@ namespace Drivers::USB::UsbDevice {
                 uint8_t xferType = ep->bmAttributes & EP_XFER_TYPE_MASK;
                 bool isIn = (ep->bEndpointAddress & EP_DIR_IN) != 0;
 
-                // HID interrupt IN endpoint
-                if (foundHid && !foundEp && isIn && xferType == EP_XFER_INTERRUPT) {
+                // HID or hub interrupt IN endpoint
+                if ((foundHid || currentHub) && !foundEp && isIn && xferType == EP_XFER_INTERRUPT) {
                     dev->InterruptEpNum     = ep->bEndpointAddress & 0x0F;
                     dev->InterruptMaxPacket = ep->wMaxPacketSize & 0x7FF;
                     dev->InterruptInterval  = ep->bInterval;
@@ -455,7 +558,7 @@ namespace Drivers::USB::UsbDevice {
         // The xHCI slot model currently stores one interface; a future model
         // can retain every alternate/interface without changing the userspace
         // claim ABI, which already names the interface number explicitly.
-        bool knownInterface = foundBt || foundMsc ||
+        bool knownInterface = foundBt || foundMsc || isHub ||
             dev->InterfaceClass == CLASS_HID;
         if (!knownInterface) {
             bool inFirstInterface = false;
@@ -502,15 +605,24 @@ namespace Drivers::USB::UsbDevice {
                                    cfgHdr.bConfigurationValue, 0, 0, nullptr, false);
         if (cc != Xhci::CC_SUCCESS) {
             KernelLogStream(ERROR, "USB") << "SET_CONFIGURATION failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
+        }
+        retryable = false;
+
+        // -----------------------------------------------------------------
+        // Step 8b: Hub descriptor (the slot context must describe the hub)
+        // -----------------------------------------------------------------
+        if (isHub && (!foundEp || !Hub::ReadDescriptor(slotId, speed, hubInfo))) {
+            KernelLogStream(WARNING, "USB") << "Slot " << (uint64_t)slotId
+                << ": hub left unconfigured";
+            isHub = false;
         }
 
         // -----------------------------------------------------------------
         // Step 9: Configure Endpoints
         // -----------------------------------------------------------------
         if (foundEp || foundBulkIn || foundBulkOut) {
-            auto* inputCtx2 = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
+            auto* inputCtx2 = (Xhci::InputContext*)Xhci::AllocateDmaPage();
 
             // Start with slot context
             inputCtx2->ICC.AddFlags = (1 << 0);
@@ -528,7 +640,7 @@ namespace Drivers::USB::UsbDevice {
                 if (dci > maxDci) maxDci = dci;
 
                 // Allocate interrupt transfer ring
-                auto* intRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+                auto* intRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
                 dev->InterruptRing        = intRing;
                 dev->InterruptRingPhys    = Memory::SubHHDM(intRing);
                 dev->InterruptRingEnqueue = 0;
@@ -542,6 +654,10 @@ namespace Drivers::USB::UsbDevice {
 
                 auto& epCtx = inputCtx2->EP[dci - 1];
                 uint32_t xhciInterval = ConvertInterval(speed, dev->InterruptInterval);
+                // Split transactions through a hub's TT: poll at least every
+                // 8 ms. Cheap, and conservative on the VL805, which Linux
+                // gives a quirk for periodic endpoints behind a TT.
+                if (dev->TtHubSlot != 0 && xhciInterval > 6) xhciInterval = 6;
                 epCtx.Field0 = (xhciInterval << 16);
                 epCtx.Field1 = (3 << 1)
                              | (Xhci::EP_TYPE_INTERRUPT_IN << 3)
@@ -556,7 +672,7 @@ namespace Drivers::USB::UsbDevice {
                 inputCtx2->ICC.AddFlags |= (1 << dci);
                 if (dci > maxDci) maxDci = dci;
 
-                auto* bulkInRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+                auto* bulkInRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
                 dev->BulkInRing        = bulkInRing;
                 dev->BulkInRingPhys    = Memory::SubHHDM(bulkInRing);
                 dev->BulkInRingEnqueue = 0;
@@ -583,7 +699,7 @@ namespace Drivers::USB::UsbDevice {
                 inputCtx2->ICC.AddFlags |= (1 << dci);
                 if (dci > maxDci) maxDci = dci;
 
-                auto* bulkOutRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+                auto* bulkOutRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
                 dev->BulkOutRing        = bulkOutRing;
                 dev->BulkOutRingPhys    = Memory::SubHHDM(bulkOutRing);
                 dev->BulkOutRingEnqueue = 0;
@@ -608,6 +724,19 @@ namespace Drivers::USB::UsbDevice {
             inputCtx2->Slot.Field0 = (inputCtx2->Slot.Field0 & ~(0x1Fu << 27))
                                    | (maxDci << 27);
 
+            // A hub: Hub flag, Number of Ports and, for a high-speed hub, its
+            // TT Think Time (xHCI 6.2.2). The controller needs them to
+            // schedule split transactions for what sits behind it.
+            if (isHub) {
+                inputCtx2->Slot.Field0 |= Xhci::SLOT_HUB;
+                inputCtx2->Slot.Field1 = (inputCtx2->Slot.Field1 & 0x00FFFFFF)
+                                       | ((uint32_t)hubInfo.Ports << 24);
+                if (speed == Xhci::SPEED_HIGH) {
+                    inputCtx2->Slot.Field2 = (inputCtx2->Slot.Field2 & ~(3u << 16))
+                                           | ((uint32_t)hubInfo.ThinkTime << 16);
+                }
+            }
+
             // Send Configure Endpoint command
             Xhci::TRB cfgTrb = {};
             uint64_t inputCtx2Phys = Memory::SubHHDM(inputCtx2);
@@ -620,8 +749,7 @@ namespace Drivers::USB::UsbDevice {
             if (cc != Xhci::CC_SUCCESS) {
                 KernelLogStream(ERROR, "USB") << "Configure Endpoint failed, slot="
                     << (uint64_t)slotId << " cc=" << (uint64_t)cc;
-                dev->Active = false;
-                return 0;
+                return false;
             }
 
             if (foundEp) {
@@ -713,6 +841,8 @@ namespace Drivers::USB::UsbDevice {
             KernelLogStream(OK, "USB") << "Slot " << (uint64_t)slotId << ": Bluetooth Adapter"
                 << " VID:" << base::hex << (uint64_t)dev->VendorId
                 << " PID:" << (uint64_t)dev->ProductId << base::dec;
+        } else if (isHub) {
+            dev->KernelDriverBound = true;
         } else if (dev->InterfaceClass == CLASS_MASS_STORAGE &&
                    dev->InterfaceSubClass == SUBCLASS_SCSI &&
                    dev->InterfaceProtocol == PROTOCOL_BULK_ONLY &&
@@ -733,7 +863,7 @@ namespace Drivers::USB::UsbDevice {
         // Publish to userspace only after endpoint configuration and kernel
         // class-driver binding decisions are complete.
         dev->Ready = true;
-        return slotId;
+        return true;
     }
 
 }

kernel/src/Drivers/USB/UsbDevice.hpp

@@ -80,6 +80,9 @@ namespace Drivers::USB::UsbDevice {
     constexpr uint8_t PROTOCOL_KEYBOARD  = 0x01;
     constexpr uint8_t PROTOCOL_MOUSE     = 0x02;
 
+    // Hub class
+    constexpr uint8_t CLASS_HUB          = 0x09;
+
     // Wireless Controller class (Bluetooth)
     constexpr uint8_t CLASS_WIRELESS     = 0xE0;
     constexpr uint8_t SUBCLASS_RF        = 0x01;
@@ -119,8 +122,24 @@ namespace Drivers::USB::UsbDevice {
     // Public API
     // ---------------------------------------------------------------------------
 
-    // Enumerate a newly connected device on the given port with the given speed.
+    // Where a device is attached: a root hub port, or a port of a hub
+    // (Xhci::UsbDeviceInfo describes the fields).
+    struct DeviceLocation {
+        uint8_t  RootPort;
+        uint8_t  ParentSlot;
+        uint8_t  ParentPort;
+        uint8_t  Depth;
+        uint32_t RouteString;
+        uint8_t  TtHubSlot;
+        uint8_t  TtPort;
+        bool     TtMulti;
+    };
+
+    // Enumerate a newly connected device on the given root port with the given speed.
     // Returns the assigned slot ID, or 0 on failure.
     uint8_t EnumerateDevice(uint8_t portId, uint32_t speed);
 
+    // Enumerate a device at `location` (a hub port that has been reset).
+    uint8_t EnumerateDevice(const DeviceLocation& location, uint32_t speed);
+
 };

kernel/src/Drivers/USB/Xhci.cpp

@@ -11,11 +11,14 @@
 #include "HidKeyboard.hpp"
 #include "HidMouse.hpp"
 #include "MassStorage.hpp"
+#include "Hub.hpp"
 #include "UserUsb.hpp"
 #include <Pci/Pci.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 #include <Memory/HHDM.hpp>
+#include <Memory/Dma.hpp>
+#include <ACPI/GsiRouting.hpp>
 #include <Memory/Paging.hpp>
 #include <Memory/PageFrameAllocator.hpp>
 #include <Libraries/Memory.hpp>
@@ -59,6 +62,21 @@ namespace Drivers::USB::Xhci {
     static std::atomic<bool> g_hotplugProcessing{false};
     static std::atomic<bool> g_irqDrainPending{false};
 
+    // HID interrupt endpoints that completed with an error. The event drain
+    // does not re-arm them (a halted endpoint takes commands to clear, which
+    // cannot be waited on from there): it records the error, and the
+    // deferred-work pass recovers the endpoint once g_hidRecoverDueMs is
+    // reached. Low/full-speed devices behind a hub's TT (every mouse on a
+    // Raspberry Pi 4) see transaction and split errors often enough that
+    // without this the device goes dead until replugged.
+    // g_hidErrorStreak counts errors since the last good report; it sets the
+    // backoff, and at HID_REENUMERATE_STREAK the device is reset and
+    // enumerated again, as a replug would.
+    static constexpr uint32_t HID_REENUMERATE_STREAK = 6;
+    static std::atomic<uint32_t> g_hidErrorCc[MAX_SLOTS + 1] = {};       // 0 = no recovery due
+    static std::atomic<uint32_t> g_hidErrorStreak[MAX_SLOTS + 1] = {};
+    static std::atomic<uint64_t> g_hidRecoverDueMs[MAX_SLOTS + 1] = {};
+
     // PollEvents also runs directly from the MSI handler. KernelLogStream uses
     // a process-context mutex, so printing there can deadlock if the interrupt
     // preempts a CPU that already owns the terminal lock. Capture the small
@@ -220,6 +238,13 @@ namespace Drivers::USB::Xhci {
         *(volatile uint32_t*)(g_opBase + reg) = value;
     }
 
+    // PollEvents() acknowledges the Port Status Change event, clearing PRC,
+    // so a controller that resets instantly (QEMU) can complete the reset
+    // between two reads of PRC. A port that left reset enabled is done too.
+    static bool PortResetComplete(uint32_t portsc) {
+        return (portsc & PORTSC_PRC) || ((portsc & PORTSC_PED) && !(portsc & PORTSC_PR));
+    }
+
     static uint32_t ReadOp(uint32_t reg) {
         return *(volatile uint32_t*)(g_opBase + reg);
     }
@@ -236,18 +261,44 @@ namespace Drivers::USB::Xhci {
         return *(volatile uint32_t*)(g_mmioBase + reg);
     }
 
+    // TRBs (and the buffers they point at) must be visible before the
+    // controller is told to look.
     static void WriteDoorbell(uint32_t index, uint32_t value) {
+        Arch::DmaWriteFence();
         *(volatile uint32_t*)(g_dbBase + index * 4) = value;
     }
 
+    // Hand a TRB to the controller: everything else in it first, then the
+    // control word that carries its cycle bit. A running ring may fetch the
+    // TRB the moment the cycle bit matches.
+    static inline void PublishTrb(TRB& trb, uint32_t control) {
+        Arch::DmaWriteFence();
+        *(volatile uint32_t*)&trb.Control = control;
+    }
+
     // -------------------------------------------------------------------------
     // DMA buffer allocation
     // -------------------------------------------------------------------------
 
+    // Whether the controller's DMA snoops the CPU caches. PCI controllers
+    // are taken to be coherent; a platform controller says so in _CCA.
+    static bool g_dmaCoherent = true;
+
+    uint8_t* AllocateDmaPage(uint64_t& outPhysAddr) {
+        return (uint8_t*)Memory::Dma::AllocatePage(outPhysAddr, g_dmaCoherent);
+    }
+
+    void* AllocateDmaPage() {
+        uint64_t phys;
+        return AllocateDmaPage(phys);
+    }
+
+    void FreeDmaPage(void* page) {
+        Memory::Dma::FreePage(page, g_dmaCoherent);
+    }
+
     static uint8_t* AllocateDmaBuffer(uint64_t& outPhysAddr) {
-        void* virt = Memory::g_pfa->AllocateZeroed();
-        outPhysAddr = Memory::SubHHDM(virt);
-        return (uint8_t*)virt;
+        return AllocateDmaPage(outPhysAddr);
     }
 
     // -------------------------------------------------------------------------
@@ -412,6 +463,24 @@ namespace Drivers::USB::Xhci {
     // MSI setup (same pattern as E1000E)
     // -------------------------------------------------------------------------
 
+    // Legacy INTx, for controllers without MSI (or platforms that cannot
+    // receive it): the line the firmware or Pci::AssignInterruptLines put
+    // in the Interrupt Line register. HandleInterrupt suits a level line:
+    // clearing IMAN.IP drops it.
+    static bool SetupIntx(uint8_t bus, uint8_t dev, uint8_t func) {
+        uint8_t line = Pci::LegacyRead8(bus, dev, func, (uint8_t)Pci::PCI_REG_INTERRUPT);
+        if (line == 0 || line >= 24) return false;
+
+        uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
+        Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND,
+                           pciCmd & ~Pci::PCI_CMD_INTX_DISABLE);
+        Arch::RegisterIrqHandler(line, HandleInterrupt);
+        Arch::EnableLegacyIrq(line);
+
+        KernelLogStream(OK, "xHCI") << "Using INTx: platform line " << base::dec << (uint64_t)line;
+        return true;
+    }
+
     static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
         uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
         if (cap == 0) {
@@ -474,6 +543,16 @@ namespace Drivers::USB::Xhci {
         BudgetExhausted,
     };
 
+    // Without MSI nothing interrupts: the deferred-work check (timer and
+    // idle loop) looks at the event ring itself instead. This is the only
+    // mode on platforms whose PCIe has no MSI controller (Raspberry Pi 4).
+    static bool g_pollMode = false;
+
+    static bool EventRingHasWork() {
+        const volatile TRB& evt = g_evtRing[g_evtRingDequeue];
+        return ((evt.Control & TRB_CYCLE_BIT) != 0) == g_evtRingCCS;
+    }
+
     static PollResult PollEventsBounded(uint32_t maxEvents) {
         // PollEvents runs from synchronous poll loops (ControlTransfer,
         // SendCommand, firmware download) and idle bottom halves on different
@@ -506,6 +585,8 @@ namespace Drivers::USB::Xhci {
             if (evtCycle != g_evtRingCCS) {
                 break; // No more events
             }
+            // The rest of the event was written before its cycle bit.
+            Arch::DmaReadFence();
 
             uint32_t trbType = (evt.Control & TRB_TYPE_MASK) >> TRB_TYPE_SHIFT;
             lastType = trbType;
@@ -528,8 +609,12 @@ namespace Drivers::USB::Xhci {
                     WriteOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE,
                             (portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
 
-                    // Defer enumeration to ProcessDeferredWork outside interrupt context.
-                    if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
+                    // Defer enumeration to ProcessDeferredWork outside interrupt
+                    // context. Changes during the boot scan are kept too (that
+                    // pass skips ports already enumerated): a device that
+                    // connects after the scan looked at its port, slower on
+                    // real hardware than in QEMU, would otherwise be lost.
+                    if (portId >= 1 && portId <= g_maxPorts) {
                         g_hotplugPending[portId - 1].store(true, std::memory_order_release);
                         g_deferredWorkPending.store(true, std::memory_order_release);
                     }
@@ -625,12 +710,17 @@ namespace Drivers::USB::Xhci {
                                 uint16_t len = (residual < reqLen)
                                     ? (uint16_t)(reqLen - residual) : 0;
 
-                                if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
+                                if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
+                                    // Re-armed once the change is handled.
+                                    Hub::OnStatusChange((uint8_t)slotId, g_interruptDataBuf[slotId],
+                                                        len, completionCode);
+                                } else if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
                                     if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
                                         HidKeyboard::ProcessReport(g_interruptDataBuf[slotId], len);
                                     } else if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
                                         HidMouse::ProcessReport(g_interruptDataBuf[slotId], len);
                                     }
+                                    g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
                                     // Re-queue for next HID report
                                     QueueInterruptTransfer(slotId);
                                 } else if (g_transferCallbacks[slotId]) {
@@ -655,6 +745,23 @@ namespace Drivers::USB::Xhci {
                                 g_diagnosticWorkPending.store(true, std::memory_order_release);
                             }
 
+                            if (dev.InterfaceClass == UsbDevice::CLASS_HUB && dev.InterruptEpNum &&
+                                epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
+                                Hub::OnStatusChange((uint8_t)slotId, nullptr, 0, completionCode);
+                            }
+
+                            if (dev.InterfaceClass == UsbDevice::CLASS_HID && dev.InterruptEpNum &&
+                                epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
+                                // Retry at once, then back off 10 ms doubling.
+                                uint32_t streak = g_hidErrorStreak[slotId].fetch_add(
+                                    1, std::memory_order_relaxed) + 1;
+                                uint64_t delay = streak <= 1 ? 0 : 10ull << (streak - 2);
+                                if (delay > 1000) delay = 1000;
+                                g_hidRecoverDueMs[slotId].store(
+                                    Timekeeping::GetMilliseconds() + delay, std::memory_order_relaxed);
+                                g_hidErrorCc[slotId].store(completionCode, std::memory_order_release);
+                            }
+
                             // Notify callback of errors too
                             if (g_transferCallbacks[slotId]) {
                                 g_transferCallbacks[slotId](slotId, epDci,
@@ -770,7 +877,7 @@ namespace Drivers::USB::Xhci {
         if (g_cmdRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        slot.Control = control;
+        PublishTrb(slot, control);
 
         // Advance enqueue pointer
         g_cmdRingEnqueue++;
@@ -819,10 +926,45 @@ namespace Drivers::USB::Xhci {
     // ControlTransfer - perform a control transfer on EP0
     // -------------------------------------------------------------------------
 
+    // A STALL or a transaction error halts EP0 (xHCI 4.10.2.1), with the
+    // rest of the TD still queued; until it is reset every later request on
+    // the slot times out. Reset Endpoint, then Set TR Dequeue past the TD.
+    static void RecoverEP0(uint8_t slotId) {
+        UsbDeviceInfo& dev = g_devices[slotId];
+
+        TRB resetTrb = {};
+        resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
+                         | ((uint32_t)slotId << 24)
+                         | (1 << 16);  // DCI=1 (EP0)
+        SendCommand(resetTrb);
+
+        uint64_t newDeq = dev.EP0RingPhys + (uint64_t)dev.EP0RingEnqueue * sizeof(TRB);
+        if (dev.EP0RingCCS) newDeq |= 1;  // DCS bit
+
+        TRB deqTrb = {};
+        deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
+        deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
+        deqTrb.Control    = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
+                          | ((uint32_t)slotId << 24)
+                          | (1 << 16);  // DCI=1 (EP0)
+        SendCommand(deqTrb);
+    }
+
+    static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
+                                          uint16_t wValue, uint16_t wIndex, uint16_t wLength,
+                                          void* data, uint64_t dataPhys, bool dirIn, bool trueNested);
+
+    // Data stages of a non-coherent controller go through this page (taken
+    // under g_controlXferLock). Callers pass small stack buffers, and cache
+    // maintenance cannot make those safe: a cache line the buffer shares with
+    // live stack data that the CPU dirties while the transfer runs is written
+    // back over what the device wrote.
+    static uint8_t* g_ctrlBounce = nullptr;
+    static uint64_t g_ctrlBouncePhys = 0;
+
     uint32_t ControlTransfer(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
                              uint16_t wValue, uint16_t wIndex, uint16_t wLength,
                              void* data, bool dirIn) {
-
         if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
             return 0xFF;
         }
@@ -836,6 +978,32 @@ namespace Drivers::USB::Xhci {
                           g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
         if (!trueNested) g_controlXferLock.Acquire();
 
+        bool hasData = data != nullptr && wLength != 0;
+        bool bounce = hasData && !g_dmaCoherent && !trueNested &&
+                      g_ctrlBounce != nullptr && wLength <= 0x1000;
+        uint32_t cc;
+        if (bounce) {
+            // Copied in both directions: bytes a short IN transfer leaves
+            // alone keep the caller's contents, as with a direct transfer.
+            memcpy(g_ctrlBounce, data, wLength);
+            cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex,
+                                       wLength, g_ctrlBounce, g_ctrlBouncePhys, dirIn, false);
+            if (dirIn) memcpy(data, g_ctrlBounce, wLength);
+        } else {
+            // The data stage buffer is the caller's (cacheable) memory.
+            if (hasData) Memory::Dma::SyncForDevice(data, wLength, g_dmaCoherent);
+            cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex, wLength,
+                                       data, hasData ? Memory::SubHHDM(data) : 0, dirIn, trueNested);
+            if (hasData && dirIn) Memory::Dma::SyncForCpu(data, wLength, g_dmaCoherent);
+        }
+
+        if (!trueNested) g_controlXferLock.Release();
+        return cc;
+    }
+
+    static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
+                                          uint16_t wValue, uint16_t wIndex, uint16_t wLength,
+                                          void* data, uint64_t dataPhys, bool dirIn, bool trueNested) {
         UsbDeviceInfo& dev = g_devices[slotId];
 
         // --- Setup Stage TRB ---
@@ -853,15 +1021,15 @@ namespace Drivers::USB::Xhci {
         if (dev.EP0RingCCS) {
             setupControl |= TRB_CYCLE_BIT;
         }
-        setup.Control = setupControl;
+        // The Setup TRB's cycle bit is published only after the Data and
+        // Status TRBs are complete (below): if EP0 is still running, it
+        // would otherwise run into a half-built TD.
 
         // Advance EP0 enqueue (handles Link TRB wrap)
         AdvanceEP0Ring(dev);
 
         // --- Data Stage TRB (if wLength > 0) ---
         if (wLength > 0 && data != nullptr) {
-            uint64_t dataPhys = Memory::SubHHDM(data);
-
             TRB& dataTrb = dev.EP0Ring[dev.EP0RingEnqueue];
             dataTrb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
             dataTrb.Parameter1 = (uint32_t)(dataPhys >> 32);
@@ -874,7 +1042,7 @@ namespace Drivers::USB::Xhci {
             if (dev.EP0RingCCS) {
                 dataControl |= TRB_CYCLE_BIT;
             }
-            dataTrb.Control = dataControl;
+            PublishTrb(dataTrb, dataControl);
 
             AdvanceEP0Ring(dev);
         }
@@ -895,10 +1063,12 @@ namespace Drivers::USB::Xhci {
         if (dev.EP0RingCCS) {
             statusControl |= TRB_CYCLE_BIT;
         }
-        status.Control = statusControl;
+        PublishTrb(status, statusControl);
 
         AdvanceEP0Ring(dev);
 
+        PublishTrb(setup, setupControl);
+
         // Ring doorbell for this slot, target EP0 (DCI 1)
         g_xferCompleted[slotId] = false;
         WriteDoorbell(slotId, 1);
@@ -922,8 +1092,9 @@ namespace Drivers::USB::Xhci {
         while (Timekeeping::GetMilliseconds() - xferStart < 2000) {
             PollEvents();
             if (g_xferCompleted[slotId]) {
-                g_controlXferLock.Release();
-                return g_xferCompletionCode[slotId];
+                uint32_t cc = g_xferCompletionCode[slotId];
+                if (cc != CC_SUCCESS && cc != CC_SHORT_PACKET) RecoverEP0(slotId);
+                return cc;
             }
             for (int j = 0; j < 100; j++) {
                 Arch::CompilerBarrier();
@@ -956,7 +1127,6 @@ namespace Drivers::USB::Xhci {
                           | (1 << 16);  // DCI=1 (EP0)
         SendCommand(deqTrb);
 
-        g_controlXferLock.Release();
         return 0xFF;
     }
 
@@ -986,7 +1156,7 @@ namespace Drivers::USB::Xhci {
         if (dev.InterruptRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        trb.Control = control;
+        PublishTrb(trb, control);
 
         // Advance enqueue (handles Link TRB wrap)
         AdvanceInterruptRing(dev);
@@ -1095,6 +1265,86 @@ namespace Drivers::USB::Xhci {
         SendCommand(deqTrb);
     }
 
+    // -------------------------------------------------------------------------
+    // ResetRootPort - reset a connected root hub port again
+    // -------------------------------------------------------------------------
+
+    bool ResetRootPort(uint8_t portId, uint32_t& speed) {
+        if (!g_initialized || portId == 0 || portId > g_maxPorts) return false;
+        uint32_t reg = OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE;
+
+        uint32_t portsc = ReadOp(reg);
+        if (!(portsc & PORTSC_CCS)) return false;
+        WriteOp(reg, (portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
+
+        bool resetDone = false;
+        for (uint32_t i = 0; i < 100000; i++) {
+            PollEvents();
+            if (PortResetComplete(ReadOp(reg))) { resetDone = true; break; }
+            for (int j = 0; j < 100; j++) Arch::CompilerBarrier();
+        }
+        if (!resetDone) return false;
+
+        portsc = ReadOp(reg);
+        WriteOp(reg, (portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
+        if (!(portsc & PORTSC_PED)) return false;
+        speed = (portsc >> 10) & 0xF;
+        return true;
+    }
+
+    // -------------------------------------------------------------------------
+    // RecoverBulkEndpoint - clear a halt or a timed-out TRB on a bulk endpoint
+    // -------------------------------------------------------------------------
+
+    void RecoverBulkEndpoint(uint8_t slotId, bool dirIn) {
+        if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
+        UsbDeviceInfo& dev = g_devices[slotId];
+        uint8_t epNum = dirIn ? dev.BulkInEpNum : dev.BulkOutEpNum;
+        TRB* ring = dirIn ? dev.BulkInRing : dev.BulkOutRing;
+        if (epNum == 0 || ring == nullptr) return;
+
+        uint8_t dci = dirIn ? (uint8_t)(epNum * 2 + 1) : (uint8_t)(epNum * 2);
+
+        // Reset Endpoint succeeds only on a halted endpoint; one that is still
+        // running (the transfer timed out) is stopped instead. Either way it
+        // ends up Stopped, which Set TR Dequeue requires.
+        TRB resetTrb = {};
+        resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
+                         | ((uint32_t)slotId << 24)
+                         | ((uint32_t)dci << 16);
+        bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
+
+        if (!wasHalted) {
+            TRB stopTrb = {};
+            stopTrb.Control = (TRB_STOP_ENDPOINT << TRB_TYPE_SHIFT)
+                            | ((uint32_t)slotId << 24)
+                            | ((uint32_t)dci << 16);
+            SendCommand(stopTrb);
+        }
+
+        uint64_t newDeq = dirIn
+            ? dev.BulkInRingPhys + (uint64_t)dev.BulkInRingEnqueue * sizeof(TRB)
+            : dev.BulkOutRingPhys + (uint64_t)dev.BulkOutRingEnqueue * sizeof(TRB);
+        if (dirIn ? dev.BulkInRingCCS : dev.BulkOutRingCCS) newDeq |= 1;  // DCS bit
+
+        TRB deqTrb = {};
+        deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
+        deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
+        deqTrb.Control    = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
+                          | ((uint32_t)slotId << 24)
+                          | ((uint32_t)dci << 16);
+        SendCommand(deqTrb);
+
+        // Reset Endpoint put the host's data toggle back to DATA0; CLEAR_FEATURE
+        // (ENDPOINT_HALT) clears the device's stall and does the same on its
+        // side (USB 2.0 9.4.5). Without a reset the toggles still agree.
+        if (wasHalted) {
+            ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
+                            0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
+                            (uint16_t)((dirIn ? 0x80 : 0x00) | epNum), 0, nullptr, false);
+        }
+    }
+
     // -------------------------------------------------------------------------
     // QueueBulkInTransfer
     // -------------------------------------------------------------------------
@@ -1123,7 +1373,7 @@ namespace Drivers::USB::Xhci {
         if (dev.BulkInRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        trb.Control = control;
+        PublishTrb(trb, control);
 
         AdvanceBulkInRing(dev);
 
@@ -1250,6 +1500,9 @@ namespace Drivers::USB::Xhci {
     // -------------------------------------------------------------------------
 
     void QueueBulkOutTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
+        if (data != nullptr && length != 0) {
+            Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
+        }
         if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
 
         UsbDeviceInfo& dev = g_devices[slotId];
@@ -1264,7 +1517,7 @@ namespace Drivers::USB::Xhci {
         if (dev.BulkOutRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        trb.Control = control;
+        PublishTrb(trb, control);
 
         AdvanceBulkOutRing(dev);
 
@@ -1277,8 +1530,23 @@ namespace Drivers::USB::Xhci {
     // BulkTransfer - blocking bulk IN/OUT transfer
     // -------------------------------------------------------------------------
 
+    static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
+                                         uint32_t length, uint32_t* actualLength);
+
     uint32_t BulkTransfer(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
                           uint32_t length, uint32_t* actualLength) {
+        if (data != nullptr && length != 0) {
+            Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
+        }
+        uint32_t cc = BulkTransferUnsynced(slotId, dirIn, data, dataPhys, length, actualLength);
+        if (data != nullptr && length != 0 && dirIn) {
+            Memory::Dma::SyncForCpu(data, length, g_dmaCoherent);
+        }
+        return cc;
+    }
+
+    static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
+                                         uint32_t length, uint32_t* actualLength) {
         if (actualLength) *actualLength = 0;
         if (length == 0) return CC_SUCCESS;
         if (length > 0x10000) return 0xFF;
@@ -1301,7 +1569,7 @@ namespace Drivers::USB::Xhci {
 
             uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
             if (dev.BulkInRingCCS) control |= TRB_CYCLE_BIT;
-            trb.Control = control;
+            PublishTrb(trb, control);
 
             AdvanceBulkInRing(dev);
             target = dev.BulkInEpNum * 2 + 1;
@@ -1315,7 +1583,7 @@ namespace Drivers::USB::Xhci {
 
             uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
             if (dev.BulkOutRingCCS) control |= TRB_CYCLE_BIT;
-            trb.Control = control;
+            PublishTrb(trb, control);
 
             AdvanceBulkOutRing(dev);
             target = dev.BulkOutEpNum * 2;
@@ -1395,10 +1663,25 @@ namespace Drivers::USB::Xhci {
         return g_initialized;
     }
 
+    // A HID endpoint whose error backoff has run out.
+    static bool HidRecoveryDue() {
+        uint64_t now = 0;
+        for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+            if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
+            if (now == 0) now = Timekeeping::GetMilliseconds();
+            if (now >= g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) return true;
+        }
+        return false;
+    }
+
     bool HasDeferredWork() {
+        if (g_initialized && g_pollMode && EventRingHasWork()) {
+            g_irqDrainPending.store(true, std::memory_order_release);
+        }
         return g_initialized &&
             (g_irqDrainPending.load(std::memory_order_acquire) ||
              g_deferredWorkPending.load(std::memory_order_acquire) ||
+             (g_bootScanComplete && (Hub::HasPendingWork() || HidRecoveryDue())) ||
              g_diagnosticWorkPending.load(std::memory_order_acquire));
     }
 
@@ -1448,7 +1731,9 @@ namespace Drivers::USB::Xhci {
     }
 
     static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
-        if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
+        if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
+            Hub::UnregisterHub(slotId);
+        } else if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
             MassStorage::UnregisterDevice(slotId);
         } else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
                    dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
@@ -1471,6 +1756,105 @@ namespace Drivers::USB::Xhci {
         }
     }
 
+    void RemoveDevice(uint8_t slotId) {
+        if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
+
+        // Children first: a hub's slot outlives the devices addressed through it.
+        for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+            if (s != slotId && g_devices[s].Active && g_devices[s].ParentSlot == slotId) {
+                RemoveDevice(s);
+            }
+        }
+
+        UserUsb::DeviceDisconnected(slotId);
+        UnregisterClassDriver(slotId, g_devices[slotId]);
+        g_devices[slotId].Active = false;
+        g_hidErrorCc[slotId].store(0, std::memory_order_relaxed);
+        g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
+        g_transferCallbacks[slotId] = nullptr;
+        DisableSlot(slotId);
+        g_dcbaa[slotId] = 0;
+        g_devices[slotId] = {};
+        KernelLogStream(INFO, "xHCI") << "Slot " << base::dec << (uint64_t)slotId << " deactivated";
+    }
+
+    // Bring a HID device's interrupt endpoint back after a failed transfer.
+    // Reset Endpoint clears a halt (a Context State Error when the error did
+    // not halt it), Set TR Dequeue skips the failed TRB, and CLEAR_FEATURE
+    // (ENDPOINT_HALT) clears a stall on the device and puts its data toggle
+    // back to DATA0 to match the reset host side (USB 2.0 9.4.5). If errors keep coming, the
+    // device is removed and its port reset, as a replug would.
+    static void RecoverHidEndpoint(uint8_t slotId, uint32_t cc) {
+        UsbDeviceInfo& dev = g_devices[slotId];
+        if (!dev.Active || dev.InterfaceClass != UsbDevice::CLASS_HID ||
+            dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
+
+        uint32_t streak = g_hidErrorStreak[slotId].load(std::memory_order_relaxed);
+        if (streak >= HID_REENUMERATE_STREAK) {
+            KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
+                << ": HID endpoint keeps failing (cc=" << (uint64_t)cc
+                << "), resetting the device";
+            uint8_t parentSlot = dev.ParentSlot, parentPort = dev.ParentPort;
+            uint8_t rootPort = dev.PortId;
+            RemoveDevice(slotId);
+            if (parentSlot != 0) {
+                Hub::RequestPortCheck(parentSlot, parentPort);
+            } else if (rootPort >= 1 && rootPort <= g_maxPorts) {
+                // Disable the port (PED is write-1-to-clear) so the hot-plug
+                // pass resets it before enumerating.
+                uint32_t reg = OP_PORTSC_BASE + (rootPort - 1) * OP_PORTSC_STRIDE;
+                WriteOp(reg, (ReadOp(reg) & PORTSC_PRESERVE) | PORTSC_PED);
+                g_hotplugPending[rootPort - 1].store(true, std::memory_order_release);
+                g_deferredWorkPending.store(true, std::memory_order_release);
+            }
+            return;
+        }
+
+        if (streak <= 1) {
+            KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
+                << ": HID transfer failed (cc=" << (uint64_t)cc << "), recovering endpoint";
+        }
+
+        uint8_t dci = dev.InterruptEpNum * 2 + 1;
+
+        TRB resetTrb = {};
+        resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
+                         | ((uint32_t)slotId << 24)
+                         | ((uint32_t)dci << 16);
+        bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
+
+        uint64_t newDeq = dev.InterruptRingPhys
+                        + (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
+        if (dev.InterruptRingCCS) newDeq |= 1;  // DCS bit
+
+        TRB deqTrb = {};
+        deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
+        deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
+        deqTrb.Control    = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
+                          | ((uint32_t)slotId << 24)
+                          | ((uint32_t)dci << 16);
+        SendCommand(deqTrb);
+
+        // Only after a real reset: the host toggle is untouched otherwise.
+        if (wasHalted) {
+            ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
+                            0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
+                            (uint16_t)(0x80 | dev.InterruptEpNum), 0, nullptr, false);
+        }
+
+        if (dev.Active) QueueInterruptTransfer(slotId);
+    }
+
+    static void ProcessHidRecovery() {
+        uint64_t now = Timekeeping::GetMilliseconds();
+        for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+            if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
+            if (now < g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) continue;
+            uint32_t cc = g_hidErrorCc[s].exchange(0, std::memory_order_acq_rel);
+            if (cc != 0) RecoverHidEndpoint(s, cc);
+        }
+    }
+
     // -------------------------------------------------------------------------
     // ProcessDeferredWork - handle hot-plug outside interrupt context.
     // Called from the BSP idle path so class-driver probing can wait on timers.
@@ -1488,7 +1872,8 @@ namespace Drivers::USB::Xhci {
 
         bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
         bool haveHotplug = g_bootScanComplete &&
-            g_deferredWorkPending.load(std::memory_order_acquire);
+            (g_deferredWorkPending.load(std::memory_order_acquire) || Hub::HasPendingWork() ||
+             HidRecoveryDue());
         if (!haveDiagnostics && !haveHotplug) return;
 
         // CAS claim: any idle core may call this, but only one runs at a time.
@@ -1505,6 +1890,9 @@ namespace Drivers::USB::Xhci {
             return;
         }
 
+        // First: a device it gives up on is queued for the hot-plug pass below.
+        ProcessHidRecovery();
+
         g_deferredWorkPending.store(false, std::memory_order_release);
 
         for (uint32_t port = 0; port < g_maxPorts; port++) {
@@ -1537,7 +1925,7 @@ namespace Drivers::USB::Xhci {
                     for (uint32_t i = 0; i < 100000; i++) {
                         PollEvents();
                         uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
-                        if (ps & PORTSC_PRC) {
+                        if (PortResetComplete(ps)) {
                             resetDone = true;
                             break;
                         }
@@ -1565,25 +1953,23 @@ namespace Drivers::USB::Xhci {
                     UsbDevice::EnumerateDevice(port + 1, speed);
                 }
             } else {
-                // Device disconnected — deactivate its slot
+                // Device disconnected: deactivate its slot (and, for a hub,
+                // the slots of everything behind it)
                 for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
-                    if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
-                        UserUsb::DeviceDisconnected(s);
-                        UnregisterClassDriver(s, g_devices[s]);
-                        g_devices[s].Active = false;
-                        g_transferCallbacks[s] = nullptr;
-                        DisableSlot(s);
-                        g_dcbaa[s] = 0;
-                        g_devices[s] = {};
-                        KernelLogStream(INFO, "xHCI") << "Hot-unplug: slot "
-                            << base::dec << (uint64_t)s << " (port "
-                            << (uint64_t)(port + 1) << ") deactivated";
+                    if (g_devices[s].Active && g_devices[s].PortId == port + 1 &&
+                        g_devices[s].ParentSlot == 0) {
+                        KernelLogStream(INFO, "xHCI") << "Hot-unplug: port "
+                            << base::dec << (uint64_t)(port + 1);
+                        RemoveDevice(s);
                         break;
                     }
                 }
             }
         }
 
+        // Port changes behind hubs.
+        Hub::ProcessPendingWork();
+
         g_hotplugProcessing.store(false, std::memory_order_release);
     }
 
@@ -1591,32 +1977,22 @@ namespace Drivers::USB::Xhci {
     // Probe (called by PCI driver matching framework)
     // -------------------------------------------------------------------------
 
-    bool Probe(const Pci::PciDevice& dev) {
-        if (g_initialized) return false;
-
-        KernelLogStream(OK, "xHCI") << "Found controller at PCI "
-            << base::hex << (uint64_t)dev.Bus << ":"
-            << (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
-
-        // Read BAR0 and map MMIO region
-        uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
-
-        KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
-
-        constexpr uint64_t MmioSize = 0x10000;
-        for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
+    // Map the register block, reset the controller and set up its
+    // context and command/event rings. Interrupt delivery is the caller's.
+    static void InitializeController(uint64_t mmioPhys, uint64_t mmioSize) {
+        for (uint64_t offset = 0; offset < mmioSize; offset += 0x1000) {
             Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
         }
 
         g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
 
-        Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
-        KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
+        // Parse capability registers. CAPLENGTH and HCIVERSION share the
+        // first dword; read it whole, since narrower accesses are not safe
+        // behind every PCIe bridge (Linux does the same).
+        uint32_t capBase = ReadCap(CAP_CAPLENGTH);
+        g_capLength = (uint8_t)(capBase & 0xFF);
 
-        // Parse capability registers
-        g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
-
-        uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
+        uint32_t hciVersion = capBase >> 16;
         KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
             << ", CapLength: " << (uint64_t)g_capLength;
 
@@ -1636,6 +2012,16 @@ namespace Drivers::USB::Xhci {
         g_rtBase = g_mmioBase + rtsOff;
         g_dbBase = g_mmioBase + dbOff;
 
+        // A firmware-described register block may be smaller than what the
+        // capability registers point into (port registers, interrupter 0,
+        // the doorbells): map those too.
+        uint64_t needed = g_capLength + OP_PORTSC_BASE + (uint64_t)((hcsParams1 >> 24) & 0xFF) * OP_PORTSC_STRIDE;
+        if (rtsOff + 0x40 > needed) needed = rtsOff + 0x40;
+        if (dbOff + 4 * ((hcsParams1 & 0xFF) + 1) > needed) needed = dbOff + 4 * ((hcsParams1 & 0xFF) + 1);
+        for (uint64_t offset = mmioSize; offset < needed; offset += 0x1000) {
+            Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
+        }
+
         KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
             << ", MaxPorts: " << (uint64_t)g_maxPorts
             << ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
@@ -1681,6 +2067,8 @@ namespace Drivers::USB::Xhci {
         // Program CONFIG
         WriteOp(OP_CONFIG, g_maxSlots);
 
+        if (!g_dmaCoherent) g_ctrlBounce = AllocateDmaBuffer(g_ctrlBouncePhys);
+
         // Allocate DCBAA
         g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
         WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
@@ -1728,12 +2116,10 @@ namespace Drivers::USB::Xhci {
         g_evtRingCCS = true;
         g_evtRingDequeue = 0;
         KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
-
-        // MSI setup
-        if (!SetupMsi(dev.Bus, dev.Device, dev.Function)) {
-            KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
     }
 
+    // Enable the interrupter, run the controller, power and enumerate ports.
+    static bool StartController() {
         // Enable interrupter 0
         WriteRt(IR0_IMAN, IMAN_IE);
         WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
@@ -1772,7 +2158,7 @@ namespace Drivers::USB::Xhci {
             for (uint32_t i = 0; i < 100000; i++) {
                 PollEvents();
                 uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
-                if (ps & PORTSC_PRC) { resetDone = true; break; }
+                if (PortResetComplete(ps)) { resetDone = true; break; }
                 for (int j = 0; j < 100; j++) Arch::CompilerBarrier();
             }
 
@@ -1801,11 +2187,75 @@ namespace Drivers::USB::Xhci {
             UsbDevice::EnumerateDevice(port + 1, speed);
         }
 
+        // Connected ports the scan found nothing on (still training, or the
+        // enumeration failed) get another look from the hot-plug path.
+        for (uint32_t port = 0; port < g_maxPorts; port++) {
+            if (!(ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE) & PORTSC_CCS)) continue;
+            bool active = false;
+            for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+                if (g_devices[s].Active && g_devices[s].PortId == port + 1) active = true;
+            }
+            if (!active) {
+                g_hotplugPending[port].store(true, std::memory_order_release);
+                g_deferredWorkPending.store(true, std::memory_order_release);
+            }
+        }
+
         g_bootScanComplete = true;
         KernelLogStream(OK, "xHCI") << "Initialization complete";
         return true;
     }
 
+    bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
+                       bool activeLow, bool dmaCoherent) {
+        if (g_initialized) return false;
+
+        g_dmaCoherent = dmaCoherent;
+        InitializeController(mmioPhys, mmioSize < 0x1000 ? 0x1000 : (mmioSize + 0xFFF) & ~0xFFFULL);
+
+        uint8_t line = gsi != 0 ? Hal::GsiRouting::LineForGsi(gsi, levelTriggered, activeLow) : 0;
+        if (line != 0) {
+            Arch::RegisterIrqHandler(line, HandleInterrupt);
+            Arch::EnableLegacyIrq(line);
+            KernelLogStream(OK, "xHCI") << "Interrupt: GSI " << base::dec << (uint64_t)gsi
+                << " on platform line " << (uint64_t)line;
+        } else {
+            KernelLogStream(WARNING, "xHCI") << "No interrupt, using poll mode";
+        }
+        // Watch the event ring from the timer as well. The interrupt is a
+        // PCI INTx routed by firmware through a bridge the kernel does not
+        // drive; if it never arrives, input still works (a ring read per
+        // tick costs next to nothing).
+        g_pollMode = true;
+
+        return StartController();
+    }
+
+    bool Probe(const Pci::PciDevice& dev) {
+        if (g_initialized) return false;
+
+        KernelLogStream(OK, "xHCI") << "Found controller at PCI "
+            << base::hex << (uint64_t)dev.Bus << ":"
+            << (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
+
+        uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
+
+        KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
+
+        Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
+        KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
+
+        InitializeController(mmioPhys, 0x10000);
+
+        // MSI setup
+        if (!SetupMsi(dev.Bus, dev.Device, dev.Function) && !SetupIntx(dev.Bus, dev.Device, dev.Function)) {
+            KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
+            g_pollMode = true;
+        }
+
+        return StartController();
+    }
+
     // -------------------------------------------------------------------------
     // Initialize (standalone fallback path)
     // -------------------------------------------------------------------------
@@ -2043,8 +2493,9 @@ namespace Drivers::USB::Xhci {
         // -----------------------------------------------------------------
         // Step 12: MSI setup
         // -----------------------------------------------------------------
-        if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function)) {
-            KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
+        if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function) && !SetupIntx(foundDev->Bus, foundDev->Device, foundDev->Function)) {
+            KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
+            g_pollMode = true;
         }
 
         // -----------------------------------------------------------------

kernel/src/Drivers/USB/Xhci.hpp

@@ -161,6 +161,7 @@ namespace Drivers::USB::Xhci {
 
     // Completion codes (from Status field bits 31:24)
     constexpr uint32_t CC_SUCCESS             = 1;
+    constexpr uint32_t CC_STALL_ERROR         = 6;
     constexpr uint32_t CC_SHORT_PACKET        = 13;
 
     // ---------------------------------------------------------------------------
@@ -177,6 +178,10 @@ namespace Drivers::USB::Xhci {
     // Device Context structures (xHCI spec section 6.2)
     // ---------------------------------------------------------------------------
 
+    // Slot Context Field0 flags
+    constexpr uint32_t SLOT_MTT        = (1u << 25);  // Multi-TT
+    constexpr uint32_t SLOT_HUB        = (1u << 26);  // the device is a hub
+
     struct SlotContext {
         uint32_t Field0;    // Route String, Speed, MTT, Hub, Context Entries
         uint32_t Field1;    // Max Exit Latency, Root Hub Port Number, Num Ports
@@ -232,8 +237,21 @@ namespace Drivers::USB::Xhci {
     struct UsbDeviceInfo {
         bool     Active;
         bool     Ready;               // descriptors/endpoints and binding are complete
-        uint8_t  PortId;
+        uint8_t  PortId;              // root hub port the device's branch hangs off
         uint32_t Speed;
+
+        // Topology (xHCI 4.3, 8.9). A device behind a hub records the hub's
+        // slot and port, its route string (one 4-bit port number per hub
+        // tier, tier 1 in bits 3:0) and, for a low/full-speed device behind a
+        // high-speed hub, the hub and port whose transaction translator
+        // carries its traffic.
+        uint8_t  ParentSlot;          // 0 = attached to a root port
+        uint8_t  ParentPort;          // port number on the parent hub
+        uint8_t  Depth;               // hubs between the device and the root port
+        uint32_t RouteString;
+        uint8_t  TtHubSlot;           // 0 = no transaction translator
+        uint8_t  TtPort;
+        bool     TtMulti;
         uint16_t VendorId;
         uint16_t ProductId;
         uint8_t  InterfaceClass;
@@ -295,6 +313,19 @@ namespace Drivers::USB::Xhci {
 
     void Initialize();
     bool Probe(const Pci::PciDevice& dev);
+
+    // A controller described by ACPI rather than found on PCI (PNP0D10,
+    // e.g. the Raspberry Pi 4's VL805 behind firmware-configured PCIe):
+    // its register block, its interrupt (an ACPI GSI, 0 = poll) and
+    // whether its DMA snoops the CPU caches (_CCA).
+    bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
+                       bool activeLow, bool dmaCoherent);
+
+    // A zeroed page for rings, contexts and buffers the controller accesses,
+    // mapped to suit the controller's DMA coherency (Memory/Dma.hpp).
+    uint8_t* AllocateDmaPage(uint64_t& outPhysAddr);
+    void* AllocateDmaPage();
+    void FreeDmaPage(void* page);
     bool IsInitialized();
     bool HasDeferredWork();
 
@@ -337,6 +368,17 @@ namespace Drivers::USB::Xhci {
     // outstanding-transfer accounting before submitting fresh work.
     void ResetBulkOutEndpoint(uint8_t slotId);
 
+    // Reset a connected root port again (an enumeration retry) and report the
+    // speed the device came up at. False if it is gone or did not enable.
+    bool ResetRootPort(uint8_t portId, uint32_t& speed);
+
+    // Bring a bulk endpoint back after a failed BulkTransfer(): clear a halt
+    // (a STALL or a transaction error) on both the host and the device, or,
+    // after a timeout, stop the endpoint and drop its stale TRB, so the next
+    // transfer neither fails nor completes against the old one. Process or
+    // boot context only (it waits on commands and EP0).
+    void RecoverBulkEndpoint(uint8_t slotId, bool dirIn);
+
     // True while PollEvents() is draining the event ring.  A command submitter
     // reached from inside an event callback must fire-and-forget (not wait),
     // since a nested PollEvents() is a no-op.
@@ -374,6 +416,10 @@ namespace Drivers::USB::Xhci {
     // Access device info
     UsbDeviceInfo* GetDevice(uint8_t slotId);
 
+    // Tear down a device that went away, and everything behind it if it is a
+    // hub: userspace claims, the class driver, the slot. Process context.
+    void RemoveDevice(uint8_t slotId);
+
     // Poll the event ring from process/idle context. The hard interrupt only
     // acknowledges and masks the source, then queues deferred draining.
     void PollEvents();

kernel/src/Efi/UEFI.hpp

@@ -276,6 +276,13 @@ namespace Efi {
     inline EFI_RESET_SYSTEM g_ResetSystem = nullptr;
 
     inline void Init(SystemTable* ST, const montauk::boot::EfiInfo& efi) {
+        /* The system table and the runtime regions are firmware-reserved
+           memory, which the HHDM does not cover on every architecture:
+           map them (identity and HHDM) before the first access. */
+        if (Memory::VMM::g_paging) {
+            Memory::VMM::g_paging->MapEfiRuntime(efi);
+        }
+
         Kt::KernelLogStream(Kt::OK, "UEFI") << "ST Minor Revision: " << ST->Header.Revision.MinorRevision;
         Kt::KernelLogStream(Kt::OK, "UEFI") << "ST Major Revision: " << ST->Header.Revision.MajorRevision;
 
@@ -284,12 +291,6 @@ namespace Efi {
         if (ST->RuntimeServices != nullptr) {
             Kt::KernelLogStream(Kt::OK, "UEFI") << "EFI Runtime Service API is available.";
 
-            /* Identity-map EFI runtime service regions so firmware code
-               can reference its own data at physical addresses */
-            if (Memory::VMM::g_paging) {
-                Memory::VMM::g_paging->MapEfiRuntime(efi);
-            }
-
             EFI_TIME Time;
             EFI_TIME_CAPABILITIES TimeCapabilities;