feat: aarch64 port - Raspberry Pi 4/400 SD boot (SDHCI, ext2 root + first-boot grow), USB hubs, PCI INTx routing, SMP, DMA fences

kernel/src/ACPI/GsiRouting.cpp

@@ -0,0 +1,41 @@
+/*
+    * GsiRouting.cpp
+    * Platform interrupt lines for ACPI global system interrupts
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "GsiRouting.hpp"
+#include <Arch/Interrupts.hpp>
+#include <CppLib/Spinlock.hpp>
+
+namespace Hal::GsiRouting {
+
+    static constexpr uint8_t FirstLine = 1;
+    static constexpr uint8_t LineCount = 24;
+
+    static uint32_t g_lineGsi[LineCount] = {};
+    static uint8_t g_nextLine = FirstLine;
+    static kcp::Mutex g_lock;
+
+    uint8_t LineForGsi(uint32_t gsi, bool levelTriggered, bool activeLow) {
+        if (Arch::PlatformLinesAreFixed()) return 0;
+
+        g_lock.Acquire();
+        for (uint8_t line = FirstLine; line < g_nextLine; line++) {
+            if (g_lineGsi[line] == gsi) {
+                g_lock.Release();
+                return line;
+            }
+        }
+
+        uint8_t line = 0;
+        if (g_nextLine < LineCount &&
+            Arch::AssignPlatformLine(g_nextLine, gsi, levelTriggered, activeLow)) {
+            line = g_nextLine++;
+            g_lineGsi[line] = gsi;
+        }
+        g_lock.Release();
+        return line;
+    }
+
+}

kernel/src/ACPI/GsiRouting.hpp

@@ -0,0 +1,19 @@
+/*
+    * GsiRouting.hpp
+    * Platform interrupt lines for ACPI global system interrupts
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+namespace Hal::GsiRouting {
+
+    // The platform line (1-23) that delivers global system interrupt `gsi`,
+    // assigning one on first use (Arch::AssignPlatformLine); every device on
+    // one GSI shares its line. Drivers register their handler on the line
+    // and call Arch::EnableLegacyIrq. Returns 0 where the platform's lines
+    // are wired by the hardware (x86) or when none is left.
+    uint8_t LineForGsi(uint32_t gsi, bool levelTriggered, bool activeLow);
+
+}

kernel/src/Api/BuildNo.hpp

@@ -12,4 +12,4 @@
 
 #pragma once
 
-#define MONTAUK_BUILD_NUMBER 37
+#define MONTAUK_BUILD_NUMBER 73

kernel/src/Api/Device.hpp

@@ -248,6 +248,10 @@ namespace montauk::abi {
         case Drivers::Storage::BLOCK_KIND_USB_MSC:
             buf->type = 4;
             break;
+        case Drivers::Storage::BLOCK_KIND_SD:
+            buf->type = 5;
+            buf->rpm = 1;
+            break;
         default:
             break;
         }

kernel/src/Api/Storage.hpp

@@ -163,6 +163,45 @@ namespace montauk::abi {
         return (int64_t)Fs::FsProbe::SyncAndUnmountAll();
     }
 
+    // Grow a mounted volume to fill the free space after its partition: the
+    // MBR entry first, only as far as the filesystem can use, then the
+    // filesystem a slice per call. Interrupted grows resume from the next
+    // call (or boot), since a partition larger than its filesystem is valid.
+    static int64_t Sys_FsGrow(int drive, FsGrowStatus* out, bool apply) {
+        int partIndex = Fs::FsProbe::GetPartitionForDrive(drive);
+        auto* part = Drivers::Storage::Gpt::GetPartition(partIndex);
+        if (part == nullptr) return UnsupportedOp;   // ramdisk, or no such drive
+
+        uint64_t maxSectors = Drivers::Storage::Gpt::GetMbrPartitionMaxSectors(partIndex);
+        if (maxSectors < part->SectorCount) maxSectors = part->SectorCount;
+
+        Fs::Vfs::GrowStatus gs{};
+        int rc = Fs::Vfs::VfsGrow(drive, maxSectors, false, &gs);
+        if (rc == -2) return UnsupportedOp;
+        if (rc < 0) return IoFailure;
+
+        // Leave whatever the filesystem cannot use unpartitioned, with the end
+        // on a 1 MiB boundary like the partitions mk-rpi4-sd.sh makes.
+        uint64_t wanted = (part->StartLba + (gs.targetBytes + 511) / 512 + 2047) / 2048 * 2048
+                        - part->StartLba;
+        if (wanted > maxSectors) wanted = maxSectors;
+
+        if (apply && wanted > part->SectorCount) {
+            if (Drivers::Storage::Gpt::ResizeMbrPartition(partIndex, wanted) != 0) return IoFailure;
+        }
+
+        rc = apply ? Fs::Vfs::VfsGrow(drive, part->SectorCount, true, &gs) : 0;
+        if (rc == -2) return UnsupportedOp;
+        if (rc < 0) return IoFailure;
+
+        out->fsBytes = gs.fsBytes;
+        out->partBytes = part->SectorCount * 512;
+        out->targetBytes = gs.targetBytes;
+        out->stepsDone = gs.stepsDone;
+        out->stepsTotal = gs.stepsTotal;
+        return rc;
+    }
+
     // Format a partition with a filesystem. Returns 0 on success, GenericFailure on error.
     static int64_t Sys_FsFormat(const FsFormatParams* params) {
         if (params == nullptr) return InvalidArgument;

kernel/src/Api/Syscall.cpp

@@ -454,6 +454,10 @@ namespace montauk::abi {
             case SYS_FS_SYNC:
                 if (!Sched::Consent::Require(CAP_STORAGE_ADMIN)) return CapabilityDenied;
                 return Sys_FsSync();
+            case SYS_FS_GROW:
+                if (!Sched::Consent::Require(CAP_STORAGE_ADMIN)) return CapabilityDenied;
+                if (!UserMemory::Range(frame->arg2, sizeof(FsGrowStatus), true)) return BadPointer;
+                return Sys_FsGrow((int)frame->arg1, (FsGrowStatus*)frame->arg2, frame->arg3 != 0);
             case SYS_FSFORMAT:
                 if (!Sched::Consent::Require(CAP_STORAGE_ADMIN)) return CapabilityDenied;
                 if (!UserMemory::Readable<FsFormatParams>(frame->arg1)) return BadPointer;

kernel/src/Api/Syscall.hpp

@@ -400,6 +400,11 @@ namespace montauk::abi {
     static constexpr uint64_t SYS_WIN_DRAG_FORMATS     = 206;  // (serial, TransferFormatInfo*, max) -> count
     static constexpr uint64_t SYS_WIN_DRAG_GET_DATA    = 207;  // (serial, mime, buf, max, uint32_t* len) -> status
 
+    /* Storage.hpp -- grow a mounted volume (and its MBR partition) to fill the
+       free space after it. apply=0 only reports; apply=1 does one bounded slice
+       and returns 1 while more remains, 0 once done. CAP_STORAGE_ADMIN. */
+    static constexpr uint64_t SYS_FS_GROW              = 208;  // (drive, FsGrowStatus*, apply) -> 1 / 0 / status
+
     /* Kernel-owned process capabilities. User identities may namespace
        per-user resources, but never participate in authorization decisions. */
     static constexpr uint64_t CAP_PROCESS_ADMIN = 1ULL << 0;
@@ -915,7 +920,7 @@ namespace montauk::abi {
 
     struct DiskInfo {
         uint8_t  port;              // block device index
-        uint8_t  type;              // 0=none, 1=SATA, 2=SATAPI, 3=NVMe, 4=USB mass storage
+        uint8_t  type;              // 0=none, 1=SATA, 2=SATAPI, 3=NVMe, 4=USB mass storage, 5=SD card
         uint8_t  sataGen;           // SATA gen (1/2/3)
         uint8_t  _pad0;
         uint64_t sectorCount;       // Total user-addressable sectors
@@ -997,6 +1002,14 @@ namespace montauk::abi {
         char     label[32];       // volume label
     };
 
+    struct FsGrowStatus {
+        uint64_t fsBytes;         // filesystem size now
+        uint64_t partBytes;       // partition size now
+        uint64_t targetBytes;     // filesystem size once grown; == fsBytes if it cannot grow
+        uint32_t stepsDone;       // progress of the grow, out of stepsTotal
+        uint32_t stepsTotal;
+    };
+
     struct ProcInfo {
         int32_t  pid;
         int32_t  parentPid;

kernel/src/Arch/Cpu.hpp

@@ -36,6 +36,15 @@ namespace Arch {
     // Full read/write memory barrier.
     inline void MemoryFence();
 
+    // Order this CPU's earlier writes to DMA memory (descriptors, buffers)
+    // before a later write the device acts on: the descriptor's valid/cycle
+    // bit, or a doorbell register. x86 keeps stores in order already.
+    inline void DmaWriteFence();
+
+    // Order the read of a device-written valid/cycle/status field before the
+    // reads of the data it guards.
+    inline void DmaReadFence();
+
     // ---- Interrupt state ----
 
     inline void EnableInterrupts();
@@ -79,6 +88,13 @@ namespace Arch {
     // Write back and invalidate all CPU caches (before power state changes).
     inline void WritebackInvalidateCaches();
 
+    // Cache maintenance for devices whose DMA does not snoop the CPU caches
+    // (ACPI _CCA = 0). Clean: write dirty lines back so the device reads
+    // what the CPU wrote. Invalidate: write back and discard, so the CPU
+    // then reads what the device wrote. No-ops where DMA is coherent (x86).
+    inline void CleanDataCache(const void* address, uint64_t size);
+    inline void InvalidateDataCache(const void* address, uint64_t size);
+
     // Make instructions just written to [address, address+size) (any
     // mapping of them) visible to instruction fetch on every CPU. A no-op
     // where instruction fetch snoops data writes (x86).

kernel/src/Arch/Interrupts.hpp

@@ -53,6 +53,17 @@ namespace Arch {
     // Unmask platform interrupt line `irq` (already routed at boot).
     void EnableLegacyIrq(uint8_t irq);
 
+    // Whether the platform interrupt lines are wired by the hardware (x86:
+    // ISA IRQs and I/O APIC pins, as the firmware programs them into PCI
+    // Interrupt Line registers). Where they are not (ARM), the PCI layer
+    // assigns lines to PCI INTx pins from ACPI _PRT with AssignPlatformLine.
+    bool PlatformLinesAreFixed();
+
+    // Make platform line `irq` (1-23) deliver global system interrupt `gsi`
+    // (ACPI numbering) with the given trigger mode and polarity, so that
+    // EnableLegacyIrq(irq) unmasks it. Only where !PlatformLinesAreFixed().
+    bool AssignPlatformLine(uint8_t irq, uint32_t gsi, bool levelTriggered, bool activeLow);
+
     // A message-signalled interrupt targeting IRQ slot `irq` on the boot
     // CPU: the device writes `data` to `address` to raise it. Program these
     // into a PCI MSI capability or MSI-X table entry. Returns false when the

kernel/src/Arch/aarch64/Bringup.hpp

@@ -0,0 +1,22 @@
+/*
+    * Bringup.hpp
+    * aarch64 per-CPU bring-up shared by the boot CPU and secondaries
+    * (private to the port)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+
+namespace Arch {
+
+    // System registers every CPU programs for itself: SP_EL1 as the stack,
+    // the exception vectors, SCTLR_EL1 policy bits, FP/SIMD access.
+    // Platform.cpp.
+    void InitializeCpuRegisters();
+
+    // Load the kernel's translation regime (MAIR, TCR, TTBR1 kernel half,
+    // TTBR0 kernel-context table) on the calling CPU. Paging::Init builds
+    // the tables and calls it on the boot CPU. Paging.cpp.
+    void ActivateKernelTranslation();
+
+}

kernel/src/Arch/aarch64/Cpu.hpp

@@ -26,6 +26,15 @@ namespace Arch {
         asm volatile("dsb sy" ::: "memory");
     }
 
+    // Devices are outer-shareable observers.
+    inline void DmaWriteFence() {
+        asm volatile("dmb oshst" ::: "memory");
+    }
+
+    inline void DmaReadFence() {
+        asm volatile("dmb oshld" ::: "memory");
+    }
+
     inline void EnableInterrupts() {
         asm volatile("msr daifclr, #2" ::: "memory");
     }
@@ -93,13 +102,38 @@ namespace Arch {
     }
 
     // The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0.
+    inline uint64_t DataCacheLineSize() {
+        uint64_t ctr;
+        asm volatile("mrs %0, ctr_el0" : "=r"(ctr));
+        return 4ULL << ((ctr >> 16) & 0xF);  // DminLine
+    }
+
+    // To the point of coherency: where non-snooping device DMA looks.
+    inline void CleanDataCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = (uint64_t)address & ~(line - 1); p < end; p += line) {
+            asm volatile("dc cvac, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb sy" ::: "memory");
+    }
+
+    // Clean too: a partial line at either end may hold CPU data next to the
+    // buffer, which a plain invalidate would throw away.
+    inline void InvalidateDataCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = (uint64_t)address & ~(line - 1); p < end; p += line) {
+            asm volatile("dc civac, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb sy" ::: "memory");
+    }
+
     // Clean the data cache to the point of unification, then invalidate
     // the whole instruction cache: invalidating by address would only
     // reach the alias it names on an aliasing (VIPT) instruction cache.
     inline void SyncInstructionCache(const void* address, uint64_t size) {
-        uint64_t ctr;
-        asm volatile("mrs %0, ctr_el0" : "=r"(ctr));
-        uint64_t line = 4ULL << ((ctr >> 16) & 0xF);  // DminLine
+        uint64_t line = DataCacheLineSize();
         uint64_t start = (uint64_t)address & ~(line - 1);
         uint64_t end = (uint64_t)address + size;
         for (uint64_t p = start; p < end; p += line) {

kernel/src/Arch/aarch64/Interrupts.cpp

@@ -17,8 +17,8 @@
 //   24-45 (MSI slots)    SPIs of the GICv2m MSI frame, one per slot
 //   IRQ_TLB_SHOOTDOWN,
 //   IRQ_RESCHEDULE       SGIs 0 and 1
-// The PC platform lines (1-23) have no ARM equivalent; RouteLegacyIrq
-// refuses them and drivers fall back to MSI or polling.
+//   1-23 (platform lines) SPIs the PCI layer assigns to INTx pins from
+//                        ACPI _PRT (AssignPlatformLine)
 
 using namespace Kt;
 
@@ -114,6 +114,12 @@ namespace Arch {
 
     static Hal::ACPI::CommonSDTHeader* g_xsdt = nullptr;
 
+    struct PlatformLine {
+        uint32_t gsi;       // 0: unassigned
+        bool edgeTriggered;
+    };
+    static PlatformLine g_platformLines[24] = {};
+
     static uint64_t g_msiFramePhys = 0;
     static uint32_t g_msiSpiBase = 0;
     static uint32_t g_msiSpiCount = 0;
@@ -338,11 +344,25 @@ namespace Arch {
             << base::dec << (uint64_t)irq;
     }
 
-    // TODO(aarch64): PCI INTx needs the pin's _PRT GSI and a GSI-based
-    // routing call (RouteSpi with level/polarity); drivers poll until then.
+    bool PlatformLinesAreFixed() {
+        return false;
+    }
+
+    // ACPI GSIs are GIC INTIDs. The GIC knows only active-high levels and
+    // rising edges; an active-low line is inverted on its way in.
+    bool AssignPlatformLine(uint8_t irq, uint32_t gsi, bool levelTriggered, bool) {
+        if (irq == 0 || irq >= 24 || gsi < 32 || gsi >= g_intidCount) return false;
+        g_platformLines[irq] = PlatformLine{ gsi, !levelTriggered };
+        return true;
+    }
+
     void RouteLegacyIrq(uint8_t irq) {
-        KernelLogStream(WARNING, "IRQ") << "aarch64: no PC interrupt line "
-            << base::dec << (uint64_t)irq << " to route";
+        if (irq >= 24 || g_platformLines[irq].gsi == 0) {
+            KernelLogStream(WARNING, "IRQ") << "aarch64: platform line "
+                << base::dec << (uint64_t)irq << " is not assigned";
+            return;
+        }
+        RouteSpi(g_platformLines[irq].gsi, irq, g_platformLines[irq].edgeTriggered);
     }
 
     void EnableLegacyIrq(uint8_t irq) {

kernel/src/Arch/aarch64/Paging.cpp

@@ -10,6 +10,7 @@
 #include <Arch/Cpu.hpp>
 #include <Common/Panic.hpp>
 #include <CppLib/Spinlock.hpp>
+#include <Arch/aarch64/Bringup.hpp>
 
 // 4 KiB granule, 48-bit virtual addresses, four levels (0-3). The kernel
 // half is translated through TTBR1_EL1 (g_kernelHalfRoot), which never
@@ -223,6 +224,50 @@ namespace Memory::VMM {
         FlushTLB();
     }
 
+}
+
+namespace Arch {
+
+    // The new tables map the code running here at the same address as the
+    // bootloader's did, so the switch is a register write followed by a
+    // full invalidate.
+    void ActivateKernelTranslation() {
+        using namespace Memory::VMM;
+
+        // 48-bit VAs in both halves, 4 KiB granules, write-back walks of
+        // inner-shareable tables; the physical size is what the CPU has.
+        uint64_t mmfr0;
+        asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0));
+        uint64_t ips = mmfr0 & 0xF;
+        if (ips > 5) ips = 5;  // 52-bit output needs LPA2 descriptors
+        uint64_t tcr = 16ULL                          // T0SZ
+                     | 1ULL << 8 | 1ULL << 10        // IRGN0, ORGN0: write-back
+                     | 3ULL << 12                    // SH0: inner shareable
+                     | 0ULL << 14                    // TG0: 4 KiB
+                     | 16ULL << 16                   // T1SZ
+                     | 1ULL << 24 | 1ULL << 26       // IRGN1, ORGN1
+                     | 3ULL << 28                    // SH1
+                     | 2ULL << 30                    // TG1: 4 KiB
+                     | ips << 32;                    // IPS
+
+        asm volatile(
+            "dsb ish\n\t"
+            "msr mair_el1, %0\n\t"
+            "msr tcr_el1, %1\n\t"
+            "isb\n\t"
+            "msr ttbr1_el1, %2\n\t"
+            "msr ttbr0_el1, %3\n\t"
+            "isb\n\t"
+            "tlbi vmalle1\n\t"
+            "dsb nsh\n\t"
+            "isb"
+            :: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(g_paging->KernelRoot()) : "memory");
+    }
+
+}
+
+namespace Memory::VMM {
+
     Paging::Paging() {
         rootPhys = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
         g_kernelHalfRoot = Memory::SubHHDM(Memory::g_pfa->AllocateZeroed());
@@ -285,36 +330,7 @@ namespace Memory::VMM {
             }
         }
 
-        // 48-bit VAs in both halves, 4 KiB granules, write-back walks of
-        // inner-shareable tables; the physical size is what the CPU has.
-        uint64_t mmfr0;
-        asm volatile("mrs %0, id_aa64mmfr0_el1" : "=r"(mmfr0));
-        uint64_t ips = mmfr0 & 0xF;
-        if (ips > 5) ips = 5;  // 52-bit output needs LPA2 descriptors
-        uint64_t tcr = 16ULL                          // T0SZ
-                     | 1ULL << 8 | 1ULL << 10        // IRGN0, ORGN0: write-back
-                     | 3ULL << 12                    // SH0: inner shareable
-                     | 0ULL << 14                    // TG0: 4 KiB
-                     | 16ULL << 16                   // T1SZ
-                     | 1ULL << 24 | 1ULL << 26       // IRGN1, ORGN1
-                     | 3ULL << 28                    // SH1
-                     | 2ULL << 30                    // TG1: 4 KiB
-                     | ips << 32;                    // IPS
-
-        // The new tables map the code running here at the same address, so
-        // the switch is a register write followed by a full invalidate.
-        asm volatile(
-            "dsb ish\n\t"
-            "msr mair_el1, %0\n\t"
-            "msr tcr_el1, %1\n\t"
-            "isb\n\t"
-            "msr ttbr1_el1, %2\n\t"
-            "msr ttbr0_el1, %3\n\t"
-            "isb\n\t"
-            "tlbi vmalle1is\n\t"
-            "dsb ish\n\t"
-            "isb"
-            :: "r"(MairValue), "r"(tcr), "r"(g_kernelHalfRoot), "r"(rootPhys) : "memory");
+        Arch::ActivateKernelTranslation();
 
         Kt::KernelLogStream(Kt::OK, "VMM") << "Switched to kernel translation tables (HHDM limit 0x"
             << base::hex << g_hhdmLimit << ")";

kernel/src/Arch/aarch64/Platform.cpp

@@ -8,6 +8,7 @@
 #include <Arch/Cpu.hpp>
 #include <Arch/Smp.hpp>
 #include <Arch/aarch64/Gic.hpp>
+#include <Arch/aarch64/Bringup.hpp>
 #include <ACPI/FADT.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
@@ -57,10 +58,7 @@ namespace Arch {
         KernelLogStream(OK, "PSCI") << "PSCI through " << (g_psci == PsciConduit::Hvc ? "HVC" : "SMC");
     }
 
-    void EarlyInitialize() {
-        // No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
-        asm volatile("msr tpidr_el1, xzr");
-
+    void InitializeCpuRegisters() {
         // Run on SP_EL1 so exceptions taken from the kernel arrive on the
         // stack it is already using (Vectors.S has no SP_EL0 handlers).
         asm volatile("mov x9, sp\n\tmsr spsel, #1\n\tmov sp, x9" ::: "x9", "memory");
@@ -85,6 +83,13 @@ namespace Arch {
         asm volatile("msr cpacr_el1, %0\n\tisb" :: "r"(cpacr));
     }
 
+    void EarlyInitialize() {
+        // No per-CPU data yet: Smp::TryGetCurrentCpuData must see null.
+        asm volatile("msr tpidr_el1, xzr");
+
+        InitializeCpuRegisters();
+    }
+
     // Installed by EarlyInitialize.
     void InitializeTraps() {
         KernelLogStream(OK, "Arch") << "Installed exception vectors at 0x" << base::hex

kernel/src/Arch/aarch64/Smp.cpp

@@ -1,13 +1,19 @@
 /*
     * Smp.cpp
-    * aarch64 per-CPU data (implements the arch half of Arch/Smp.hpp)
+    * aarch64 per-CPU data and secondary CPU start-up (implements the arch half of Arch/Smp.hpp)
     * Copyright (c) 2026 Daniel Hammer
 */
 
 #include <Arch/Smp.hpp>
+#include <Arch/Cpu.hpp>
+#include <Arch/Syscall.hpp>
+#include <Arch/aarch64/Bringup.hpp>
+#include <Arch/aarch64/Gic.hpp>
+#include <Boot/BootInfo.hpp>
 #include <Libraries/Memory.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
+#include <Timekeeping/Timer.hpp>
 
 using namespace Kt;
 
@@ -47,14 +53,83 @@ namespace Smp {
             << bsp.arch.mpidr << ")";
     }
 
+    // A secondary CPU, woken by the bootloader on its own stack and still
+    // on the bootloader's translation tables. The kernel stashed this CPU's
+    // CpuData in extraArgument.
+    static void ApEntry(montauk::boot::BootCpu* bootCpu) {
+        CpuData* cpu = (CpuData*)bootCpu->extraArgument;
+
+        Arch::InitializeCpuRegisters();
+        Arch::ActivateKernelTranslation();
+        asm volatile("msr tpidr_el1, %0" :: "r"(cpu));
+
+        Arch::Gic::InitializeCpuInterface();
+        Arch::InitializeSyscallEntry();
+        Timekeeping::TimerInitializeAP();
+
+        __atomic_store_n(&cpu->started, true, __ATOMIC_RELEASE);
+        Arch::EnableInterrupts();
+
+        // Runnable work arrives as a reschedule IPI; keep the periodic tick
+        // masked while idle.
+        for (;;) {
+            Timekeeping::TimerEnterApIdle();
+            Timekeeping::ServiceDeferredWork();
+            Arch::DisableInterrupts();
+            Arch::HaltWithInterruptsDisabled();
+            Arch::EnableInterrupts();
+        }
+    }
+
     void BootAPs(const montauk::boot::SmpInfo& smp) {
-        if (smp.cpuCount > 1) {
-            g_detectedCpuCount = (int)smp.cpuCount;
+        if (smp.cpus == nullptr || smp.startCpu == nullptr || smp.cpuCount <= 1) {
+            KernelLogStream(INFO, "SMP") << "Single CPU system - no secondary CPUs to start";
+            return;
+        }
+        g_detectedCpuCount = (int)smp.cpuCount;
+
+        if (!Arch::Gic::IsInitialized()) {
+            KernelLogStream(WARNING, "SMP") << "No interrupt controller: secondary CPUs left parked";
+            return;
         }
 
-        // TODO(aarch64): start secondaries through smp.startCpu once the GIC
-        // (reschedule/TLB-shootdown SGIs) and the per-CPU generic timer work.
-        KernelLogStream(WARNING, "SMP") << "aarch64: secondary CPUs not started ("
-            << base::dec << smp.cpuCount << " reported)";
+        int apIndex = 1;
+        for (uint64_t i = 0; i < smp.cpuCount && apIndex < MaxCPUs; i++) {
+            montauk::boot::BootCpu& info = smp.cpus[i];
+            if (info.hardwareId == smp.bspHardwareId) continue;
+
+            CpuData& ap = g_cpus[apIndex];
+            ap.selfPtr = (uint64_t)&ap;
+            ap.cpuIndex = apIndex;
+            ap.currentSlot = -1;
+            ap.started = false;
+            ap.arch.mpidr = info.hardwareId;
+
+            info.extraArgument = (uint64_t)&ap;
+            // Publish the CpuData before the CPU that reads it is woken.
+            asm volatile("dsb ish" ::: "memory");
+            smp.startCpu(&info, ApEntry);
+            apIndex++;
+        }
+        g_cpuCount = apIndex;
+
+        int online = 1;
+        for (int i = 1; i < g_cpuCount; i++) {
+            uint64_t start = Arch::ReadTimestamp();
+            uint64_t frequency;
+            asm volatile("mrs %0, cntfrq_el0" : "=r"(frequency));
+            while (!__atomic_load_n(&g_cpus[i].started, __ATOMIC_ACQUIRE) &&
+                   Arch::ReadTimestamp() - start < frequency) {
+                Arch::Pause();
+            }
+            if (g_cpus[i].started) {
+                online++;
+            } else {
+                KernelLogStream(ERROR, "SMP") << "CPU " << base::dec << (uint64_t)i
+                    << " (MPIDR " << base::hex << g_cpus[i].arch.mpidr << ") failed to start";
+            }
+        }
+
+        KernelLogStream(OK, "SMP") << base::dec << (uint64_t)online << " CPU(s) online";
     }
 }

kernel/src/Arch/aarch64/Timer.cpp

@@ -92,7 +92,6 @@ namespace Arch::Timer {
 
         ReadGtdt();
         Gic::SetAckHook(IRQ_TIMER, Rearm);
-        Gic::RoutePpi(g_intid, IRQ_TIMER, g_edgeTriggered);
         StartPeriodic(periodMs);
 
         KernelLogStream(OK, "Timer") << "Generic timer: " << base::dec << frequency
@@ -100,7 +99,9 @@ namespace Arch::Timer {
         return true;
     }
 
+    // PPIs are banked: each CPU enables its own timer interrupt.
     void StartPeriodic(uint32_t periodMs) {
+        Gic::RoutePpi(g_intid, IRQ_TIMER, g_edgeTriggered);
         uint64_t period = g_ticksPerMs * periodMs;
         Smp::GetCurrentCpuData()->arch.timerPeriod = period;
         WriteCval(ReadTimestamp() + period);

kernel/src/Arch/x86_64/Apic/Interrupts.cpp

@@ -58,6 +58,14 @@ namespace Arch {
         Hal::IoApic::UnmaskIrq(Hal::IoApic::GetGsiForIrq(irq));
     }
 
+    bool PlatformLinesAreFixed() {
+        return true;
+    }
+
+    bool AssignPlatformLine(uint8_t, uint32_t, bool, bool) {
+        return false;
+    }
+
     bool ComposeMsi(uint8_t irq, MsiMessage& out) {
         // Physical destination mode, fixed delivery, edge triggered, to the
         // BSP (APIC ID 0 -- the destination field is left zero, as every

kernel/src/Arch/x86_64/Cpu.hpp

@@ -24,6 +24,15 @@ namespace Arch {
         asm volatile("mfence" ::: "memory");
     }
 
+    // Stores are not reordered with other stores, nor loads with loads.
+    inline void DmaWriteFence() {
+        asm volatile("" ::: "memory");
+    }
+
+    inline void DmaReadFence() {
+        asm volatile("" ::: "memory");
+    }
+
     inline void EnableInterrupts() {
         asm volatile("sti" ::: "memory");
     }
@@ -82,6 +91,12 @@ namespace Arch {
     inline void SyncInstructionCache(const void*, uint64_t) {
     }
 
+    inline void CleanDataCache(const void*, uint64_t) {
+    }
+
+    inline void InvalidateDataCache(const void*, uint64_t) {
+    }
+
     inline void FlushTlbPage(uint64_t virtualAddress) {
         asm volatile("invlpg (%0)" :: "r"(virtualAddress) : "memory");
     }

kernel/src/Boot/Main.cpp

@@ -138,6 +138,8 @@ extern "C" void kmain() {
         // power control on the answer.
         Hal::AML::InitializeNamespaceObjects();
 
+        Pci::AssignInterruptLines();
+
         Drivers::ProbeEarly();
         Drivers::InitializeGraphics();
 

kernel/src/Drivers/Audio/IntelHda.cpp

@@ -191,7 +191,8 @@ namespace Drivers::Audio::IntelHda {
         // Write verb to CORB
         g_corb[wp] = verb;
 
-        // Update write pointer
+        // Update write pointer once the verb is visible
+        Arch::DmaWriteFence();
         Write16(REG_CORBWP, wp);
 
         return true;
@@ -202,6 +203,7 @@ namespace Drivers::Audio::IntelHda {
             uint16_t wp = Read16(REG_RIRBWP) & 0xFF;
 
             if (g_rirbReadPtr != wp) {
+                Arch::DmaReadFence();
                 g_rirbReadPtr = (g_rirbReadPtr + 1) % g_rirbSize;
 
                 uint32_t ex = g_rirb[g_rirbReadPtr].ResponseEx;

kernel/src/Drivers/Init.cpp

@@ -14,11 +14,16 @@
 #include <Drivers/USB/Xhci.hpp>
 #include <Drivers/Storage/Ahci.hpp>
 #include <Drivers/Storage/Nvme.hpp>
+#include <Drivers/Storage/Sdhci.hpp>
 #include <Drivers/Storage/Gpt.hpp>
 #include <Drivers/Audio/IntelHda.hpp>
 #include <Graphics/Framebuffer.hpp>
 #include <Net/Net.hpp>
 #include <Terminal/Terminal.hpp>
+#include <ACPI/AcpiDevices.hpp>
+#include <Boot/Boot.hpp>
+#include <ACPI/AML/AmlInterpreter.hpp>
+#include <ACPI/AML/AmlResource.hpp>
 
 namespace Drivers {
 
@@ -89,6 +94,10 @@ namespace Drivers {
         return Storage::Nvme::Probe(dev);
     }
 
+    static bool ProbeSdhci(const Pci::PciDevice& dev) {
+        return Storage::Sdhci::Probe(dev);
+    }
+
     static bool ProbeIntelHda(const Pci::PciDevice& dev) {
         return Audio::IntelHda::Probe(dev);
     }
@@ -189,6 +198,18 @@ namespace Drivers {
         //          also reports 0x01, but it exposes an I/O-space BAR0:
         //          ReadBar0 returns 0 for I/O BARs and Probe rejects it
         //          cleanly, so matching it here is harmless.
+        // SD host controller (standard interface) - class 0x08/0x05
+        {
+            "SDHCI",
+            0,
+            0x08,                           // ClassCode (Base system peripheral)
+            0x05,                           // SubClass (SD host controller)
+            0xFF,
+            nullptr,
+            0,
+            Pci::ProbePhase::Normal,
+            ProbeSdhci,
+        },
         {
             "IntelHDA",
             0x8086,                         // VendorId (Intel)
@@ -227,8 +248,117 @@ namespace Drivers {
         Graphics::DgpuPower::PowerDownUnused();
     }
 
+    // A device's _DMA (or its parent's) bounds the addresses it can reach.
+    // DMA buffers come from anywhere in RAM, so RAM past the bound is a
+    // configuration error: on the Raspberry Pi 4 the firmware's "Limit RAM
+    // to 3 GB" setting must stay on.
+    static void WarnIfRamBeyondDma(const char* devicePath) {
+        char path[160];
+        int len = 0;
+        while (devicePath[len] != '\0' && len < 120) { path[len] = devicePath[len]; len++; }
+
+        Hal::AML::ResourceList dma;
+        bool found = false;
+        for (int attempt = 0; attempt < 2 && !found; attempt++) {
+            const char suffix[] = "._DMA";
+            for (int i = 0; i < (int)sizeof(suffix); i++) path[len + i] = suffix[i];
+            Hal::AML::Object obj;
+            if (Hal::AML::GetInterpreter().EvaluateObject(path, obj) &&
+                obj.Type == Hal::AML::ObjectType::Buffer &&
+                Hal::AML::ParseResourceTemplate(obj.Buffer.Data, obj.Buffer.Length, dma)) {
+                found = true;
+                break;
+            }
+            // Retry on the parent device.
+            while (len > 0 && path[len - 1] != '.') len--;
+            if (len == 0) return;
+            len--;
+        }
+
+        uint64_t dmaBase = 0, dmaLength = 0;
+        if (!found || !dma.GetMemory(dmaBase, dmaLength) || dmaLength == 0) return;
+        uint64_t dmaEnd = dmaBase + dmaLength;
+
+        const auto& map = montauk::boot::Info().memoryMap;
+        for (uint64_t i = 0; i < map.count; i++) {
+            const auto& r = map.regions[i];
+            if (r.kind == montauk::boot::MemoryKind::Usable && r.base + r.length > dmaEnd) {
+                Kt::KernelLogStream(Kt::WARNING, "Drivers") << devicePath << " can only reach RAM below 0x"
+                    << base::hex << dmaEnd << ", but there is RAM up to 0x" << (r.base + r.length)
+                    << ": its DMA may fail (Raspberry Pi 4: keep 'Limit RAM to 3 GB' enabled)";
+                return;
+            }
+        }
+    }
+
+    // An xHCI controller ACPI describes as a platform device (PNP0D10)
+    // instead of a PCI function: the Raspberry Pi 4/400's VL805 under UEFI
+    // firmware in ACPI mode, where firmware has set up the non-ECAM PCIe
+    // behind it. Its DMA is non-coherent unless _CCA says otherwise.
+    static void ProbePlatformXhci() {
+        if (USB::Xhci::IsInitialized()) return;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("PNP0D10", info) &&
+            !Hal::AcpiDevices::FindDeviceByHid("PNP0D15", info)) {
+            return;
+        }
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            Kt::KernelLogStream(Kt::WARNING, "Drivers") << "xHCI " << info.Path
+                << ": no register block in _CRS";
+            return;
+        }
+
+        uint32_t gsi = 0;
+        bool level = true, activeLow = false;
+        if (auto* irq = resources.FindFirst(Hal::AML::ResourceType::ExtendedIrq)) {
+            gsi = irq->ExtendedIrq.Interrupt;
+            level = (irq->ExtendedIrq.Flags & 1) == 0;
+            activeLow = (irq->ExtendedIrq.Flags & 2) != 0;
+        }
+
+        bool coherent = false;
+        Hal::AML::Object cca;
+        if (Hal::AML::GetInterpreter().EvaluateChild(info.NodeIndex, "_CCA", nullptr, 0, cca) &&
+            cca.Type == Hal::AML::ObjectType::Integer) {
+            coherent = cca.Integer != 0;
+        }
+
+        Kt::KernelLogStream(Kt::INFO, "Drivers") << "xHCI platform device " << info.Path
+            << " at 0x" << base::hex << base << ", GSI " << base::dec << (uint64_t)gsi
+            << (coherent ? ", coherent DMA" : ", non-coherent DMA");
+        WarnIfRamBeyondDma(info.Path);
+        USB::Xhci::ProbePlatform(base, length, gsi, level, activeLow, coherent);
+    }
+
+    // SD host controllers ACPI describes directly: the Raspberry Pi 4's
+    // EMMC2 (BRCME88C, the SD card slot) or a standard PNP0D40.
+    static void ProbePlatformSdhci() {
+        if (Storage::Sdhci::IsInitialized()) return;
+
+        Hal::AcpiDevices::DeviceInfo info{};
+        if (!Hal::AcpiDevices::FindDeviceByHid("BRCME88C", info) &&
+            !Hal::AcpiDevices::FindDeviceByHid("PNP0D40", info)) {
+            return;
+        }
+
+        Hal::AML::ResourceList resources;
+        uint64_t base = 0, length = 0;
+        if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
+            !resources.GetMemory(base, length) || base == 0) {
+            return;
+        }
+        Storage::Sdhci::ProbePlatform(base, info.Path);
+    }
+
     void ProbeNormal() {
         Pci::ProbeAll(g_driverTable, g_driverTableCount, Pci::ProbePhase::Normal);
+        ProbePlatformXhci();
+        ProbePlatformSdhci();
     }
 
     void InitializeNetwork() {

kernel/src/Drivers/Net/E1000.cpp

@@ -14,6 +14,7 @@
 #include <Libraries/Memory.hpp>
 #include <Arch/Interrupts.hpp>
 #include <CppLib/Spinlock.hpp>
+#include <Arch/Cpu.hpp>
 #include <atomic>
 
 using namespace Kt;
@@ -491,6 +492,7 @@ namespace Drivers::Net::E1000 {
 
         // Advance the tail pointer (tells the NIC there's a new packet)
         g_txTail = (g_txTail + 1) % TX_DESC_COUNT;
+        Arch::DmaWriteFence();
         WriteReg(REG_TDT, g_txTail);
 
         g_txPacketCount++;
@@ -545,6 +547,7 @@ namespace Drivers::Net::E1000 {
             uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
             RxDescriptor& desc = g_rxDescs[nextIdx];
             if (!(desc.Status & RXSTA_DD)) break;
+            Arch::DmaReadFence();
 
             uint16_t length = desc.Length;
             g_rxPacketCount++;
@@ -554,6 +557,7 @@ namespace Drivers::Net::E1000 {
             desc.Length = 0;
             desc.Errors = 0;
             g_rxTail = nextIdx;
+            Arch::DmaWriteFence();
             WriteReg(REG_RDT, g_rxTail);
             processed++;
         }

kernel/src/Drivers/Net/E1000E.cpp

@@ -667,6 +667,7 @@ namespace Drivers::Net::E1000E {
             if (!(desc.Status & RXSTA_DD)) {
                 break;
             }
+            Arch::DmaReadFence();
 
             uint16_t length = desc.Length;
             g_rxPacketCount++;
@@ -680,6 +681,7 @@ namespace Drivers::Net::E1000E {
             desc.Errors = 0;
 
             g_rxTail = nextIdx;
+            Arch::DmaWriteFence();
             WriteReg(REG_RDT, g_rxTail);
             processed++;
         }
@@ -709,6 +711,7 @@ namespace Drivers::Net::E1000E {
         desc.Status = 0;
 
         g_txTail = (g_txTail + 1) % TX_DESC_COUNT;
+        Arch::DmaWriteFence();
         WriteReg(REG_TDT, g_txTail);
 
         g_txPacketCount++;

kernel/src/Drivers/Storage/Ahci.cpp

@@ -267,13 +267,15 @@ namespace Drivers::Storage::Ahci {
             }
         }
 
-        // Issue the command
+        // Issue the command once its header, table and PRDs are visible
+        Arch::DmaWriteFence();
         WritePortReg(port, PORT_CI, 1u << slot);
 
         // Wait for completion
         for (int i = 0; i < 5000000; i++) {
             uint32_t ci = ReadPortReg(port, PORT_CI);
             if (!(ci & (1u << slot))) {
+                Arch::DmaReadFence();
                 // Check for errors
                 uint32_t is = ReadPortReg(port, PORT_IS);
                 if (is & PORT_IS_TFES) {
@@ -641,10 +643,19 @@ namespace Drivers::Storage::Ahci {
         for (int i = 0; i < MAX_PORTS; i++) {
             if (!(g_portsImplemented & (1u << i))) continue;
 
+            // The signature arrives in a D2H FIS, which an HBA may only
+            // latch into PxSIG once FIS receive is running (QEMU does so):
+            // start every port with a device on its link, then classify.
+            if ((ReadPortReg(i, PORT_SSTS) & SSTS_DET_MASK) != SSTS_DET_PRESENT) continue;
+            InitPort(i);
+
             PortType type = ClassifyPort(i);
             g_ports[i].Type = type;
 
-            if (type == PortType::None) continue;
+            if (type == PortType::None) {
+                StopPort(i);
+                continue;
+            }
 
             const char* typeStr = "Unknown";
             switch (type) {
@@ -658,8 +669,6 @@ namespace Drivers::Storage::Ahci {
             KernelLogStream(INFO, "AHCI") << "Port " << base::dec << (uint64_t)i
                 << ": " << typeStr << " device detected";
 
-            InitPort(i);
-
             if (type == PortType::Sata) {
                 if (IdentifyDevice(i)) {
                     g_ports[i].Active = true;

kernel/src/Drivers/Storage/BlockDevice.hpp

@@ -17,6 +17,7 @@ namespace Drivers::Storage {
         BLOCK_KIND_SATAPI  = 2,
         BLOCK_KIND_NVME    = 3,
         BLOCK_KIND_USB_MSC = 4,
+        BLOCK_KIND_SD      = 5,
     };
 
     struct BlockDevice {

kernel/src/Drivers/Storage/Gpt.cpp

@@ -107,6 +107,8 @@ namespace Drivers::Storage::Gpt {
                 return GUID_LINUX_SWAP;
             case 0x83:
                 return GUID_LINUX_FS;
+            case 0xEF:
+                return GUID_EFI_SYSTEM;
             default:
                 return GUID_BASIC_DATA;
         }
@@ -551,6 +553,78 @@ namespace Drivers::Storage::Gpt {
         return removed;
     }
 
+    // -------------------------------------------------------------------------
+    // MBR partition resize
+    // -------------------------------------------------------------------------
+
+    // Read sector 0 into mbr and return the primary slot that holds `part`, or
+    // -1 when the device is GPT (protective MBR) or `part` is not a primary
+    // entry (a logical partition inside an extended one).
+    static int FindMbrSlot(const PartitionInfo& part, ProtectiveMbr& mbr) {
+        const BlockDevice* dev = GetBlockDevice(part.BlockDevIndex);
+        if (!dev || !dev->ReadSectors(dev->Ctx, 0, 1, &mbr)) return -1;
+        if (mbr.Signature != 0xAA55 || ValidateProtectiveMbr((const uint8_t*)&mbr)) return -1;
+
+        for (int i = 0; i < 4; i++) {
+            const MbrPartitionEntry& e = mbr.Partitions[i];
+            if (e.Type == 0 || IsExtendedMbrType(e.Type)) continue;
+            if (e.LbaFirst == part.StartLba && e.SectorCount == part.SectorCount) return i;
+        }
+        return -1;
+    }
+
+    uint64_t GetMbrPartitionMaxSectors(int index) {
+        const PartitionInfo* part = GetPartition(index);
+        if (!part) return 0;
+
+        ProtectiveMbr mbr;
+        if (FindMbrSlot(*part, mbr) < 0) return 0;
+        const BlockDevice* dev = GetBlockDevice(part->BlockDevIndex);
+
+        // Stop at whichever primary entry starts next (an extended entry covers
+        // every logical partition inside it), else at the end of the device.
+        uint64_t end = dev->SectorCount;
+        for (int i = 0; i < 4; i++) {
+            const MbrPartitionEntry& e = mbr.Partitions[i];
+            if (e.Type == 0 || e.SectorCount == 0) continue;
+            if (e.LbaFirst > part->StartLba && e.LbaFirst < end) end = e.LbaFirst;
+        }
+
+        // The entry stores a 32-bit sector count; keep the end addressable too.
+        if (end > 0xFFFFFFFFull) end = 0xFFFFFFFFull;
+        return end > part->StartLba ? end - part->StartLba : 0;
+    }
+
+    int ResizeMbrPartition(int index, uint64_t sectorCount) {
+        if (index < 0 || index >= g_partitionCount) return -1;
+        PartitionInfo& part = g_partitions[index];
+        if (sectorCount < part.SectorCount) return -1;
+        if (sectorCount > GetMbrPartitionMaxSectors(index)) return -1;
+        if (sectorCount == part.SectorCount) return 0;
+
+        ProtectiveMbr mbr;
+        int slot = FindMbrSlot(part, mbr);
+        if (slot < 0) return -1;
+
+        MbrPartitionEntry& e = mbr.Partitions[slot];
+        e.SectorCount = (uint32_t)sectorCount;
+        // CHS cannot address past 8 GiB; FE FF FF tells CHS-era tools to use
+        // the LBA fields instead.
+        e.ChsLast[0] = 0xFE;
+        e.ChsLast[1] = 0xFF;
+        e.ChsLast[2] = 0xFF;
+
+        const BlockDevice* dev = GetBlockDevice(part.BlockDevIndex);
+        if (!dev->WriteSectors(dev->Ctx, 0, 1, &mbr)) return -1;
+        if (dev->Flush) dev->Flush(dev->Ctx);
+
+        KernelLogStream(OK, "MBR") << "Partition " << index << " grown from "
+            << part.SectorCount << " to " << sectorCount << " sectors";
+        part.SectorCount = sectorCount;
+        part.EndLba = part.StartLba + sectorCount - 1;
+        return 0;
+    }
+
     // -------------------------------------------------------------------------
     // ASCII to UTF-16LE for partition names
     // -------------------------------------------------------------------------

kernel/src/Drivers/Storage/Gpt.hpp

@@ -127,6 +127,18 @@ namespace Drivers::Storage::Gpt {
     // hot-pluggable block device disappears.
     int RemovePartitionsForBlockDevice(int blockDevIndex);
 
+    // Largest size, in sectors, that partition `index` could be grown to: up to
+    // the next partition on its device or the end of the device, within the
+    // 32-bit LBA fields of an MBR entry. Returns 0 if the partition is not a
+    // primary entry of a plain MBR disk (GPT disks and logical partitions are
+    // not resized).
+    uint64_t GetMbrPartitionMaxSectors(int index);
+
+    // Grow MBR partition `index` in place to sectorCount sectors, rewriting its
+    // table entry and the in-memory record. Data is untouched. Returns 0 on
+    // success, -1 if the partition cannot be resized to that size.
+    int ResizeMbrPartition(int index, uint64_t sectorCount);
+
     // Look up the human-readable name for a partition type GUID.
     const char* GetTypeName(const Guid& typeGuid);
 

kernel/src/Drivers/Storage/Nvme.cpp

@@ -87,7 +87,9 @@ namespace Drivers::Storage::Nvme {
     //            CQ y Head Doorbell offset = 0x1000 + ((2y+1) * doorbellStride)
     // -------------------------------------------------------------------------
 
+    // The submission queue entry must be visible before the doorbell.
     static void WriteSqTailDoorbell(uint16_t queueId, uint16_t value) {
+        Arch::DmaWriteFence();
         uint32_t offset = 0x1000 + (2 * queueId) * g_doorbellStride;
         *(volatile uint32_t*)(g_mmioBase + offset) = value;
     }
@@ -152,6 +154,8 @@ namespace Drivers::Storage::Nvme {
             uint16_t status = cqe->Status;
 
             if ((status & CQE_PHASE_BIT) == phase) {
+                // The rest of the entry (and the data) before the phase bit.
+                Arch::DmaReadFence();
                 uint16_t consumedId = cqe->CommandId;
                 uint32_t result = cqe->Result;
                 uint16_t sqHead = cqe->SqHead;

kernel/src/Drivers/Storage/Sdhci.cpp

@@ -0,0 +1,410 @@
+/*
+    * Sdhci.cpp
+    * SD Host Controller (SDHCI) driver: SD cards as block devices
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Sdhci.hpp"
+#include "BlockDevice.hpp"
+#include <Arch/Cpu.hpp>
+#include <CppLib/Spinlock.hpp>
+#include <CppLib/Stream.hpp>
+#include <Libraries/Memory.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Terminal/Terminal.hpp>
+#include <Timekeeping/Timer.hpp>
+
+// One slot, polled, programmed I/O through the buffer data port. PIO keeps
+// the driver free of DMA address translation and cache coherency, which on
+// the Raspberry Pi 4 depend on the SoC stepping (EMMC2's _DMA).
+//
+// Every register access is 32 bits wide: the Broadcom controllers (the Pi's
+// EMMC2, and the Arasan-derived EMMC before it) do not support narrower
+// ones. 8/16-bit fields are written by read-modify-write of their dword,
+// and the transfer-mode/command pair in a single write that issues the
+// command.
+
+using namespace Kt;
+
+namespace Drivers::Storage::Sdhci {
+
+    // Registers (SD Host Controller Simplified Specification 3.00)
+    static constexpr uint32_t REG_BLOCK_SIZE_COUNT = 0x04;  // size [15:0], count [31:16]
+    static constexpr uint32_t REG_ARGUMENT         = 0x08;
+    static constexpr uint32_t REG_XFER_MODE_CMD    = 0x0C;  // transfer mode [15:0], command [31:16]
+    static constexpr uint32_t REG_RESPONSE         = 0x10;  // 4 dwords
+    static constexpr uint32_t REG_BUFFER_DATA      = 0x20;
+    static constexpr uint32_t REG_PRESENT_STATE    = 0x24;
+    static constexpr uint32_t REG_HOST_CONTROL     = 0x28;  // host ctl 1 [7:0], power [15:8]
+    static constexpr uint32_t REG_CLOCK_CONTROL    = 0x2C;  // clock [15:0], timeout [23:16], reset [31:24]
+    static constexpr uint32_t REG_INT_STATUS       = 0x30;  // normal [15:0], error [31:16]
+    static constexpr uint32_t REG_INT_ENABLE       = 0x34;  // status enables
+    static constexpr uint32_t REG_INT_SIGNAL       = 0x38;  // interrupt signal enables
+    static constexpr uint32_t REG_CAPABILITIES     = 0x40;
+    static constexpr uint32_t REG_VERSION          = 0xFC;  // host version [31:16]
+
+    // Present state
+    static constexpr uint32_t PS_CMD_INHIBIT = 1u << 0;
+    static constexpr uint32_t PS_DAT_INHIBIT = 1u << 1;
+
+    // Interrupt status
+    static constexpr uint32_t INT_CMD_COMPLETE  = 1u << 0;
+    static constexpr uint32_t INT_XFER_COMPLETE = 1u << 1;
+    static constexpr uint32_t INT_WRITE_READY   = 1u << 4;
+    static constexpr uint32_t INT_READ_READY    = 1u << 5;
+    static constexpr uint32_t INT_ERROR         = 1u << 15;
+    static constexpr uint32_t INT_ERROR_MASK    = 0xFFFF0000u;
+
+    // Host control 1 / power control
+    static constexpr uint32_t HC_4BIT       = 1u << 1;
+    static constexpr uint32_t PWR_ON        = 1u << 8;
+    static constexpr uint32_t PWR_330       = 7u << 9;
+    static constexpr uint32_t PWR_300       = 6u << 9;
+    static constexpr uint32_t PWR_180       = 5u << 9;
+
+    // Clock control
+    static constexpr uint32_t CLK_INT_EN     = 1u << 0;
+    static constexpr uint32_t CLK_INT_STABLE = 1u << 1;
+    static constexpr uint32_t CLK_SD_EN      = 1u << 2;
+    static constexpr uint32_t TIMEOUT_MAX    = 0xEu << 16;
+
+    // Software reset (byte 0x2F)
+    static constexpr uint32_t RESET_ALL = 1u << 24;
+    static constexpr uint32_t RESET_CMD = 1u << 25;
+    static constexpr uint32_t RESET_DAT = 1u << 26;
+
+    // Capabilities
+    static constexpr uint32_t CAP_VOLT_330 = 1u << 24;
+    static constexpr uint32_t CAP_VOLT_300 = 1u << 25;
+
+    // Transfer mode
+    static constexpr uint32_t XM_BLOCK_COUNT_EN = 1u << 1;
+    static constexpr uint32_t XM_AUTO_CMD12     = 1u << 2;
+    static constexpr uint32_t XM_READ           = 1u << 4;
+    static constexpr uint32_t XM_MULTI_BLOCK    = 1u << 5;
+
+    // Command register flags
+    enum Response : uint32_t {
+        RespNone  = 0,
+        Resp136   = 1,            // R2
+        Resp48    = 2 | 3u << 3,  // R1, R6, R7: CRC and index checked
+        Resp48Busy= 3 | 3u << 3,  // R1b
+        Resp48NoCrc = 2,          // R3
+    };
+    static constexpr uint32_t CMD_DATA = 1u << 5;
+
+    static constexpr uint32_t BlockSize = 512;
+    // The block count register is 16 bits wide.
+    static constexpr uint32_t MaxBlocksPerCommand = 0xFFFF;
+    static constexpr uint64_t CommandTimeoutMs = 200;
+    static constexpr uint64_t DataTimeoutMs = 1000;
+
+    // Base clock when the capabilities leave it to the platform: the
+    // Raspberry Pi 4's EMMC2 runs from a 100 MHz clock the firmware sets up.
+    static constexpr uint32_t FallbackBaseClockHz = 100000000;
+
+    static volatile uint8_t* g_regs = nullptr;
+    static bool g_initialized = false;
+    static uint32_t g_version = 0;      // 0: 1.00, 1: 2.00, 2: 3.00
+    static uint32_t g_baseClockHz = 0;
+    static uint32_t g_rca = 0;
+    static bool g_highCapacity = false;
+    static uint64_t g_sectorCount = 0;
+    static kcp::Mutex g_lock;
+
+    static inline uint32_t Read(uint32_t reg) {
+        return *(volatile uint32_t*)(g_regs + reg);
+    }
+
+    static inline void Write(uint32_t reg, uint32_t value) {
+        *(volatile uint32_t*)(g_regs + reg) = value;
+    }
+
+    static bool WaitClear(uint32_t reg, uint32_t mask, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Read(reg) & mask) {
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
+            Arch::Pause();
+        }
+        return true;
+    }
+
+    static bool Reset(uint32_t which) {
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | which);
+        return WaitClear(REG_CLOCK_CONTROL, which, 100);
+    }
+
+    // Wait for any of `bits` in the normal status, or an error. Clears what
+    // it returns on.
+    static bool WaitStatus(uint32_t bits, uint64_t timeoutMs) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            uint32_t status = Read(REG_INT_STATUS);
+            if (status & (INT_ERROR | INT_ERROR_MASK)) {
+                Write(REG_INT_STATUS, status);
+                Reset(RESET_CMD | RESET_DAT);
+                return false;
+            }
+            if (status & bits) {
+                Write(REG_INT_STATUS, status & bits);
+                return true;
+            }
+            if (Timekeeping::GetMilliseconds() - start > timeoutMs) {
+                Reset(RESET_CMD | RESET_DAT);
+                return false;
+            }
+            Arch::Pause();
+        }
+    }
+
+    static bool SendCommand(uint32_t index, uint32_t argument, uint32_t response,
+                            uint32_t transferMode = 0, bool data = false) {
+        uint32_t inhibit = PS_CMD_INHIBIT;
+        if (data || response == Resp48Busy) inhibit |= PS_DAT_INHIBIT;
+        if (!WaitClear(REG_PRESENT_STATE, inhibit, CommandTimeoutMs)) {
+            Reset(RESET_CMD | RESET_DAT);
+            return false;
+        }
+
+        Write(REG_INT_STATUS, 0xFFFFFFFF);
+        Write(REG_ARGUMENT, argument);
+        uint32_t command = index << 8 | response | (data ? CMD_DATA : 0);
+        Write(REG_XFER_MODE_CMD, command << 16 | transferMode);
+
+        if (!WaitStatus(INT_CMD_COMPLETE, CommandTimeoutMs)) return false;
+        if (response == Resp48Busy && !WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs)) return false;
+        return true;
+    }
+
+    // An application command: CMD55 with the card's address first.
+    static bool SendAppCommand(uint32_t index, uint32_t argument, uint32_t response) {
+        return SendCommand(55, g_rca << 16, Resp48) && SendCommand(index, argument, response);
+    }
+
+    // Bits [start, start+length) of a 136-bit response (CID/CSD numbering).
+    // The controller drops the CRC byte, so CSD bit n is response bit n-8.
+    static uint32_t ResponseBits(const uint32_t response[4], int start, int length) {
+        uint32_t value = 0;
+        for (int i = 0; i < length; i++) {
+            int bit = start + i - 8;
+            if (response[bit / 32] & (1u << (bit % 32))) value |= 1u << i;
+        }
+        return value;
+    }
+
+    static bool SetClock(uint32_t hz) {
+        uint32_t clock = Read(REG_CLOCK_CONTROL);
+        Write(REG_CLOCK_CONTROL, clock & ~(0xFFFFu));
+
+        uint32_t divider = 0;   // base clock / (2 * divider); 0 = undivided
+        if (hz < g_baseClockHz) {
+            divider = (g_baseClockHz + 2 * hz - 1) / (2 * hz);
+            uint32_t max = g_version >= 2 ? 0x3FF : 0x80;
+            if (divider > max) divider = max;
+            if (g_version < 2) {
+                // Before 3.00 the divider is a power of two.
+                uint32_t p = 1;
+                while (p < divider) p <<= 1;
+                divider = p;
+            }
+        }
+
+        uint32_t field = (divider & 0xFF) << 8 | ((divider >> 8) & 0x3) << 6;
+        clock = (Read(REG_CLOCK_CONTROL) & ~0xFFFFu) | field | CLK_INT_EN;
+        Write(REG_CLOCK_CONTROL, clock);
+
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (!(Read(REG_CLOCK_CONTROL) & CLK_INT_STABLE)) {
+            if (Timekeeping::GetMilliseconds() - start > 20) return false;
+            Arch::Pause();
+        }
+        Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | CLK_SD_EN);
+        return true;
+    }
+
+    static bool InitializeCard() {
+        uint32_t response[4];
+
+        if (!SendCommand(0, 0, RespNone)) return false;
+
+        // CMD8: a 2.00+ card echoes the check pattern.
+        bool v2 = SendCommand(8, 0x1AA, Resp48) && (Read(REG_RESPONSE) & 0xFFF) == 0x1AA;
+
+        // ACMD41 until the card leaves its busy state (up to a second).
+        uint32_t ocr = 0;
+        uint64_t start = Timekeeping::GetMilliseconds();
+        for (;;) {
+            if (!SendAppCommand(41, (v2 ? 1u << 30 : 0) | 0x00FF8000, Resp48NoCrc)) return false;
+            ocr = Read(REG_RESPONSE);
+            if (ocr & (1u << 31)) break;
+            if (Timekeeping::GetMilliseconds() - start > 1000) return false;
+        }
+        g_highCapacity = (ocr & (1u << 30)) != 0;
+
+        if (!SendCommand(2, 0, Resp136)) return false;
+        if (!SendCommand(3, 0, Resp48)) return false;
+        g_rca = Read(REG_RESPONSE) >> 16;
+
+        if (!SendCommand(9, g_rca << 16, Resp136)) return false;
+        for (int i = 0; i < 4; i++) response[i] = Read(REG_RESPONSE + i * 4);
+        if (ResponseBits(response, 126, 2) == 1) {
+            uint64_t cSize = ResponseBits(response, 48, 22);
+            g_sectorCount = (cSize + 1) * 1024;
+        } else {
+            uint64_t cSize = ResponseBits(response, 62, 12);
+            uint64_t mult = ResponseBits(response, 47, 3);
+            uint64_t readBlockLength = ResponseBits(response, 80, 4);
+            g_sectorCount = ((cSize + 1) << (mult + 2)) * (1ULL << readBlockLength) / BlockSize;
+        }
+
+        if (!SendCommand(7, g_rca << 16, Resp48Busy)) return false;
+        if (!g_highCapacity && !SendCommand(16, BlockSize, Resp48)) return false;
+
+        // 4-bit bus when the card agrees; 1-bit otherwise.
+        if (SendAppCommand(6, 2, Resp48)) {
+            Write(REG_HOST_CONTROL, Read(REG_HOST_CONTROL) | HC_4BIT);
+        }
+
+        SetClock(25000000);
+        return true;
+    }
+
+    // Move one block through the buffer data port. Caller buffers are only
+    // byte aligned in general, so unaligned ones go through a bounce copy.
+    static void PioBlock(uint8_t* data, bool write) {
+        uint32_t bounce[BlockSize / 4];
+        bool aligned = ((uintptr_t)data & 3) == 0;
+        uint32_t* words = aligned ? (uint32_t*)data : bounce;
+        if (write && !aligned) memcpy(bounce, data, BlockSize);
+        for (uint32_t i = 0; i < BlockSize / 4; i++) {
+            if (write) Write(REG_BUFFER_DATA, words[i]);
+            else words[i] = Read(REG_BUFFER_DATA);
+        }
+        if (!write && !aligned) memcpy(data, bounce, BlockSize);
+    }
+
+    // One command for `count` consecutive blocks: CMD17/CMD24 for a single
+    // block, else CMD18/CMD25 with the controller sending the closing CMD12
+    // itself (Auto CMD12; the BCM2711's EMMC2 wants it for multi-block reads).
+    // Cards program a multi-block write a flash page at a time; issued one
+    // block per command, each 512 bytes costs a full page program, which made
+    // writes (the first-boot grow above all) take tens of minutes.
+    static bool TransferBlocks(uint64_t lba, uint32_t count, uint8_t* data, bool write) {
+        uint32_t address = g_highCapacity ? (uint32_t)lba : (uint32_t)(lba * BlockSize);
+        bool multi = count > 1;
+        Write(REG_BLOCK_SIZE_COUNT, count << 16 | BlockSize);
+
+        uint32_t mode = write ? 0 : XM_READ;
+        if (multi) mode |= XM_BLOCK_COUNT_EN | XM_AUTO_CMD12 | XM_MULTI_BLOCK;
+        uint32_t command = write ? (multi ? 25 : 24) : (multi ? 18 : 17);
+
+        bool ok = SendCommand(command, address, Resp48, mode, true);
+        for (uint32_t i = 0; ok && i < count; i++) {
+            ok = WaitStatus(write ? INT_WRITE_READY : INT_READ_READY, DataTimeoutMs);
+            if (ok) PioBlock(data + (uint64_t)i * BlockSize, write);
+        }
+        ok = ok && WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs);
+
+        // A failed multi-block transfer leaves the card sending or receiving
+        // data; stop it so the next command finds it in the transfer state.
+        if (!ok && multi) SendCommand(12, 0, Resp48Busy);
+        return ok;
+    }
+
+    static bool TransferSectors(uint64_t lba, uint32_t count, uint8_t* data, bool write) {
+        if (lba + count > g_sectorCount) return false;
+        g_lock.Acquire();
+        bool ok = true;
+        while (ok && count > 0) {
+            uint32_t run = count > MaxBlocksPerCommand ? MaxBlocksPerCommand : count;
+            ok = TransferBlocks(lba, run, data, write);
+            lba += run;
+            data += (uint64_t)run * BlockSize;
+            count -= run;
+        }
+        g_lock.Release();
+        return ok;
+    }
+
+    static bool ReadSectors(void*, uint64_t lba, uint32_t count, void* buffer) {
+        return TransferSectors(lba, count, (uint8_t*)buffer, false);
+    }
+
+    static bool WriteSectors(void*, uint64_t lba, uint32_t count, const void* buffer) {
+        return TransferSectors(lba, count, (uint8_t*)buffer, true);
+    }
+
+    static bool InitializeController(uint64_t mmioPhys, const char* name) {
+        if (g_initialized) return false;
+
+        Memory::VMM::g_paging->MapMMIO(mmioPhys & ~0xFFFULL, Memory::HHDM(mmioPhys & ~0xFFFULL));
+        g_regs = (volatile uint8_t*)Memory::HHDM(mmioPhys);
+
+        if (!Reset(RESET_ALL)) {
+            KernelLogStream(ERROR, "SDHCI") << name << ": reset timed out";
+            return false;
+        }
+
+        g_version = (Read(REG_VERSION) >> 16) & 0xFF;
+        uint32_t caps = Read(REG_CAPABILITIES);
+        uint32_t baseMhz = g_version >= 2 ? (caps >> 8) & 0xFF : (caps >> 8) & 0x3F;
+        g_baseClockHz = baseMhz != 0 ? baseMhz * 1000000 : FallbackBaseClockHz;
+
+        uint32_t voltage = (caps & CAP_VOLT_330) ? PWR_330 : (caps & CAP_VOLT_300) ? PWR_300 : PWR_180;
+        Write(REG_HOST_CONTROL, (Read(REG_HOST_CONTROL) & ~0xFF00u) | voltage | PWR_ON);
+
+        Write(REG_INT_ENABLE, 0xFFFFFFFF);
+        Write(REG_INT_SIGNAL, 0);
+        Write(REG_CLOCK_CONTROL, (Read(REG_CLOCK_CONTROL) & ~0xFF0000u) | TIMEOUT_MAX);
+
+        if (!SetClock(400000)) {
+            KernelLogStream(ERROR, "SDHCI") << name << ": clock did not stabilize";
+            return false;
+        }
+
+        // The card needs 74 clocks after power-up before its first command.
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Timekeeping::GetMilliseconds() - start < 2) Arch::Pause();
+
+        KernelLogStream(INFO, "SDHCI") << name << ": version " << base::dec
+            << (uint64_t)(g_version + 1) << ".00, base clock " << (uint64_t)(g_baseClockHz / 1000000)
+            << " MHz" << (baseMhz == 0 ? " (assumed)" : "");
+
+        if (!InitializeCard()) {
+            KernelLogStream(INFO, "SDHCI") << name << ": no SD card";
+            return false;
+        }
+
+        g_initialized = true;
+        KernelLogStream(OK, "SDHCI") << name << ": " << (g_highCapacity ? "SDHC/SDXC" : "SDSC")
+            << " card, " << base::dec << (g_sectorCount * BlockSize / (1024 * 1024)) << " MiB";
+
+        BlockDevice dev = {};
+        dev.ReadSectors = ReadSectors;
+        dev.WriteSectors = WriteSectors;
+        dev.SectorCount = g_sectorCount;
+        dev.SectorSize = BlockSize;
+        dev.Kind = BLOCK_KIND_SD;
+        const char model[] = "SD card";
+        memcpy(dev.Model, model, sizeof(model));
+        RegisterBlockDevice(dev);
+        return true;
+    }
+
+    bool Probe(const Pci::PciDevice& dev) {
+        uint64_t bar = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
+        if (bar == 0) return false;
+        Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
+        return InitializeController(bar, "PCI controller");
+    }
+
+    bool ProbePlatform(uint64_t mmioPhys, const char* name) {
+        return InitializeController(mmioPhys, name);
+    }
+
+    bool IsInitialized() {
+        return g_initialized;
+    }
+
+}

kernel/src/Drivers/Storage/Sdhci.hpp

@@ -0,0 +1,22 @@
+/*
+    * Sdhci.hpp
+    * SD Host Controller (SDHCI) driver: SD cards as block devices
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+#include <Pci/Pci.hpp>
+
+namespace Drivers::Storage::Sdhci {
+
+    // A standard SD host controller on PCI (class 08h/05h).
+    bool Probe(const Pci::PciDevice& dev);
+
+    // One ACPI describes (the Raspberry Pi 4's EMMC2, BRCME88C; a generic
+    // PNP0D40 controller): its register block.
+    bool ProbePlatform(uint64_t mmioPhys, const char* name);
+
+    bool IsInitialized();
+
+}

kernel/src/Drivers/USB/Hub.cpp

@@ -0,0 +1,371 @@
+/*
+    * Hub.cpp
+    * USB hub class driver (USB 2.0 and SuperSpeed hubs)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Hub.hpp"
+#include "Xhci.hpp"
+#include "UsbDevice.hpp"
+#include <Arch/Cpu.hpp>
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
+#include <Timekeeping/Timer.hpp>
+#include <atomic>
+
+using namespace Kt;
+
+static void BusyWaitMs(uint64_t ms) {
+    if (Arch::InterruptsEnabled()) {
+        uint64_t start = Timekeeping::GetMilliseconds();
+        while (Timekeeping::GetMilliseconds() - start < ms) {
+            Arch::Pause();
+        }
+    } else {
+        // The tick is not advancing: count ~1 us delays instead
+        for (uint64_t i = 0; i < ms * 1000; i++) {
+            Arch::IoDelay();
+        }
+    }
+}
+
+namespace Drivers::USB::Hub {
+
+    // -------------------------------------------------------------------------
+    // Hub class requests and port status (USB 2.0 11.24, USB 3.2 10.16)
+    // -------------------------------------------------------------------------
+
+    constexpr uint8_t REQTYPE_HUB_IN     = 0xA0;  // Dev-to-host, class, device
+    constexpr uint8_t REQTYPE_HUB_OUT    = 0x20;  // Host-to-dev, class, device
+    constexpr uint8_t REQTYPE_PORT_IN    = 0xA3;  // Dev-to-host, class, other (port)
+    constexpr uint8_t REQTYPE_PORT_OUT   = 0x23;  // Host-to-dev, class, other (port)
+
+    constexpr uint8_t REQ_GET_STATUS     = 0x00;
+    constexpr uint8_t REQ_CLEAR_FEATURE  = 0x01;
+    constexpr uint8_t REQ_SET_FEATURE    = 0x03;
+    constexpr uint8_t REQ_GET_DESCRIPTOR = 0x06;
+    constexpr uint8_t REQ_SET_HUB_DEPTH  = 0x0C;  // SuperSpeed hubs only
+
+    constexpr uint8_t DESC_HUB           = 0x29;
+    constexpr uint8_t DESC_SS_HUB        = 0x2A;
+
+    // Port feature selectors
+    constexpr uint16_t FEAT_PORT_RESET   = 4;
+    constexpr uint16_t FEAT_PORT_POWER   = 8;
+
+    // wPortStatus
+    constexpr uint16_t PS_CONNECTION     = (1 << 0);
+    constexpr uint16_t PS_ENABLE         = (1 << 1);
+    constexpr uint16_t PS_RESET          = (1 << 4);
+    constexpr uint16_t PS_LOW_SPEED      = (1 << 9);   // USB 2 hubs
+    constexpr uint16_t PS_HIGH_SPEED     = (1 << 10);  // USB 2 hubs
+
+    // wPortChange
+    constexpr uint16_t PC_CONNECTION     = (1 << 0);
+    constexpr uint16_t PC_RESET          = (1 << 4);
+
+    // The route string holds a 4-bit port number per tier, five tiers.
+    constexpr uint8_t MAX_TIERS          = 5;
+    constexpr uint8_t MAX_ROUTABLE_PORT  = 15;
+
+    // -------------------------------------------------------------------------
+    // State
+    // -------------------------------------------------------------------------
+
+    struct HubState {
+        bool              Active;
+        HubDescriptorInfo Info;
+    };
+
+    static HubState g_hubs[Xhci::MAX_SLOTS + 1] = {};
+
+    // Written from the event-ring drain, consumed by ProcessPendingWork.
+    static std::atomic<uint32_t> g_changedPorts[Xhci::MAX_SLOTS + 1] = {};  // bit n = port n
+    static std::atomic<bool>     g_rearm[Xhci::MAX_SLOTS + 1] = {};
+    static std::atomic<bool>     g_endpointFailed[Xhci::MAX_SLOTS + 1] = {};
+    static std::atomic<bool>     g_pending{false};
+
+    // -------------------------------------------------------------------------
+    // Requests
+    // -------------------------------------------------------------------------
+
+    static bool Succeeded(uint32_t cc) {
+        return cc == Xhci::CC_SUCCESS || cc == Xhci::CC_SHORT_PACKET;
+    }
+
+    static bool GetPortStatus(uint8_t hubSlot, uint8_t port, uint16_t& status, uint16_t& change) {
+        uint8_t buf[4] = {};
+        uint32_t cc = Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_IN, REQ_GET_STATUS,
+                                            0, port, sizeof(buf), buf, true);
+        if (!Succeeded(cc)) return false;
+        status = (uint16_t)(buf[0] | (buf[1] << 8));
+        change = (uint16_t)(buf[2] | (buf[3] << 8));
+        return true;
+    }
+
+    static uint32_t SetPortFeature(uint8_t hubSlot, uint8_t port, uint16_t feature) {
+        return Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_OUT, REQ_SET_FEATURE,
+                                     feature, port, 0, nullptr, false);
+    }
+
+    static uint32_t ClearPortFeature(uint8_t hubSlot, uint8_t port, uint16_t feature) {
+        return Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_OUT, REQ_CLEAR_FEATURE,
+                                     feature, port, 0, nullptr, false);
+    }
+
+    // Acknowledge every change bit in `change`. Bits 0-4 clear with
+    // C_PORT_CONNECTION..C_PORT_RESET (16-20); SuperSpeed hubs add
+    // C_BH_PORT_RESET, C_PORT_LINK_STATE and C_PORT_CONFIG_ERROR.
+    static void ClearPortChanges(uint8_t hubSlot, uint8_t port, uint16_t change, bool superSpeed) {
+        for (uint16_t bit = 0; bit <= 4; bit++) {
+            if (change & (1u << bit)) ClearPortFeature(hubSlot, port, (uint16_t)(16 + bit));
+        }
+        if (superSpeed) {
+            if (change & (1u << 5)) ClearPortFeature(hubSlot, port, 29);
+            if (change & (1u << 6)) ClearPortFeature(hubSlot, port, 25);
+            if (change & (1u << 7)) ClearPortFeature(hubSlot, port, 26);
+        }
+    }
+
+    // Acknowledge hub-wide changes (local power, over-current): until they
+    // are cleared the hub keeps reporting bit 0 of its change bitmap.
+    static void ClearHubChanges(uint8_t hubSlot) {
+        uint8_t buf[4] = {};
+        uint32_t cc = Xhci::ControlTransfer(hubSlot, REQTYPE_HUB_IN, REQ_GET_STATUS,
+                                            0, 0, sizeof(buf), buf, true);
+        if (!Succeeded(cc)) return;
+        uint16_t change = (uint16_t)(buf[2] | (buf[3] << 8));
+        for (uint16_t bit = 0; bit <= 1; bit++) {  // C_HUB_LOCAL_POWER, C_HUB_OVER_CURRENT
+            if (change & (1u << bit)) {
+                Xhci::ControlTransfer(hubSlot, REQTYPE_HUB_OUT, REQ_CLEAR_FEATURE,
+                                      bit, 0, 0, nullptr, false);
+            }
+        }
+    }
+
+    bool ReadDescriptor(uint8_t slotId, uint32_t speed, HubDescriptorInfo& out) {
+        out = {};
+        out.SuperSpeed = speed >= Xhci::SPEED_SUPER;
+        uint8_t type = out.SuperSpeed ? DESC_SS_HUB : DESC_HUB;
+
+        uint8_t buf[16] = {};
+        uint32_t cc = Xhci::ControlTransfer(slotId, REQTYPE_HUB_IN, REQ_GET_DESCRIPTOR,
+                                            (uint16_t)(type << 8), 0, sizeof(buf), buf, true);
+        if (!Succeeded(cc) || buf[0] < 7 || buf[1] != type || buf[2] == 0) {
+            KernelLogStream(ERROR, "USB/Hub") << "Slot " << base::dec << (uint64_t)slotId
+                << ": hub descriptor unreadable, cc=" << (uint64_t)cc;
+            return false;
+        }
+
+        uint16_t characteristics = (uint16_t)(buf[3] | (buf[4] << 8));
+        out.Ports = buf[2];
+        out.ThinkTime = out.SuperSpeed ? 0 : (uint8_t)((characteristics >> 5) & 3);
+        out.PowerOnMs = (uint16_t)(buf[5] * 2);
+        return true;
+    }
+
+    // -------------------------------------------------------------------------
+    // Ports
+    // -------------------------------------------------------------------------
+
+    static uint8_t FindChild(uint8_t hubSlot, uint8_t port) {
+        for (uint8_t s = 1; s <= Xhci::MAX_SLOTS; s++) {
+            auto* dev = Xhci::GetDevice(s);
+            if (dev->Active && dev->ParentSlot == hubSlot && dev->ParentPort == port) return s;
+        }
+        return 0;
+    }
+
+    // Reset a connected port and report the speed the device came up at.
+    static bool ResetPort(uint8_t hubSlot, uint8_t port, bool superSpeed, uint32_t& speed) {
+        if (SetPortFeature(hubSlot, port, FEAT_PORT_RESET) != Xhci::CC_SUCCESS) return false;
+
+        uint16_t status = 0, change = 0;
+        bool done = false;
+        for (int attempt = 0; attempt < 50; attempt++) {  // ~500 ms
+            BusyWaitMs(10);
+            if (!GetPortStatus(hubSlot, port, status, change)) return false;
+            if ((change & PC_RESET) && !(status & PS_RESET)) { done = true; break; }
+            if (!(status & PS_CONNECTION)) return false;
+        }
+        ClearPortChanges(hubSlot, port, change, superSpeed);
+        if (!done || !(status & PS_ENABLE)) return false;
+
+        if (superSpeed)                   speed = Xhci::SPEED_SUPER;
+        else if (status & PS_LOW_SPEED)   speed = Xhci::SPEED_LOW;
+        else if (status & PS_HIGH_SPEED)  speed = Xhci::SPEED_HIGH;
+        else                              speed = Xhci::SPEED_FULL;
+        return true;
+    }
+
+    // Bring the port's state in line with what is attached to it: remove a
+    // device that left (or was replaced), enumerate one that arrived.
+    static void HandlePort(uint8_t hubSlot, uint8_t port, bool settled) {
+        const HubDescriptorInfo& info = g_hubs[hubSlot].Info;
+
+        uint16_t status = 0, change = 0;
+        if (!GetPortStatus(hubSlot, port, status, change)) return;
+        if (change != 0) ClearPortChanges(hubSlot, port, change, info.SuperSpeed);
+
+        bool connected = (status & PS_CONNECTION) != 0;
+        uint8_t child = FindChild(hubSlot, port);
+        if (child != 0 && (!connected || (change & PC_CONNECTION))) {
+            KernelLogStream(INFO, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
+                << " port " << (uint64_t)port << ": device disconnected";
+            Xhci::RemoveDevice(child);
+            child = 0;
+        }
+        if (!connected || child != 0) return;
+
+        if (!settled) {
+            // Connect debounce (USB 2.0 7.1.7.3): let contacts settle first.
+            BusyWaitMs(100);
+            if (!GetPortStatus(hubSlot, port, status, change)) return;
+            if (!(status & PS_CONNECTION)) return;
+        }
+
+        auto* hub = Xhci::GetDevice(hubSlot);
+        if (hub->Depth + 1 > MAX_TIERS || port > MAX_ROUTABLE_PORT) {
+            KernelLogStream(WARNING, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
+                << " port " << (uint64_t)port << ": too deep to address, ignored";
+            return;
+        }
+
+        uint32_t speed = 0;
+        if (!ResetPort(hubSlot, port, info.SuperSpeed, speed)) {
+            KernelLogStream(WARNING, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
+                << " port " << (uint64_t)port << ": reset failed";
+            return;
+        }
+        BusyWaitMs(10);  // reset recovery (USB 2.0 7.1.7.5)
+
+        UsbDevice::DeviceLocation location = {};
+        location.RootPort    = hub->PortId;
+        location.ParentSlot  = hubSlot;
+        location.ParentPort  = port;
+        location.Depth       = (uint8_t)(hub->Depth + 1);
+        location.RouteString = hub->RouteString | ((uint32_t)port << (4 * hub->Depth));
+        if (hub->Speed == Xhci::SPEED_HIGH &&
+            (speed == Xhci::SPEED_LOW || speed == Xhci::SPEED_FULL)) {
+            // This hub's transaction translator turns the high-speed
+            // traffic into low/full-speed transactions. Single-TT mode:
+            // the configuration's default alternate setting.
+            location.TtHubSlot = hubSlot;
+            location.TtPort    = port;
+            location.TtMulti   = false;
+        } else {
+            location.TtHubSlot = hub->TtHubSlot;
+            location.TtPort    = hub->TtPort;
+            location.TtMulti   = hub->TtMulti;
+        }
+
+        UsbDevice::EnumerateDevice(location, speed);
+    }
+
+    // -------------------------------------------------------------------------
+    // Public API
+    // -------------------------------------------------------------------------
+
+    void RegisterHub(uint8_t slotId, const HubDescriptorInfo& info) {
+        auto* dev = Xhci::GetDevice(slotId);
+        if (dev == nullptr || !dev->Active) return;
+
+        g_hubs[slotId].Info = info;
+        g_hubs[slotId].Active = true;
+        g_changedPorts[slotId].store(0, std::memory_order_relaxed);
+        g_rearm[slotId].store(false, std::memory_order_relaxed);
+        g_endpointFailed[slotId].store(false, std::memory_order_relaxed);
+
+        uint8_t ports = info.Ports;
+        if (ports > 31) ports = 31;  // status-change bitmap tracked in 32 bits
+
+        KernelLogStream(OK, "USB/Hub") << "Slot " << base::dec << (uint64_t)slotId << ": "
+            << (info.SuperSpeed ? "SuperSpeed" : "USB 2.0") << " hub, "
+            << (uint64_t)info.Ports << " ports, tier " << (uint64_t)(dev->Depth + 1);
+
+        if (info.SuperSpeed) {
+            Xhci::ControlTransfer(slotId, REQTYPE_HUB_OUT, REQ_SET_HUB_DEPTH,
+                                  dev->Depth, 0, 0, nullptr, false);
+        }
+
+        for (uint8_t port = 1; port <= ports; port++) {
+            SetPortFeature(slotId, port, FEAT_PORT_POWER);
+        }
+        // Power-on-to-power-good, then the connect debounce interval.
+        uint64_t wait = info.PowerOnMs < 100 ? 100 : info.PowerOnMs;
+        BusyWaitMs(wait + 100);
+
+        for (uint8_t port = 1; port <= ports && g_hubs[slotId].Active; port++) {
+            HandlePort(slotId, port, true);
+        }
+
+        if (g_hubs[slotId].Active) Xhci::QueueInterruptTransfer(slotId);
+    }
+
+    void UnregisterHub(uint8_t slotId) {
+        if (slotId == 0 || slotId > Xhci::MAX_SLOTS) return;
+        g_hubs[slotId] = {};
+        g_changedPorts[slotId].store(0, std::memory_order_relaxed);
+        g_rearm[slotId].store(false, std::memory_order_relaxed);
+        g_endpointFailed[slotId].store(false, std::memory_order_relaxed);
+    }
+
+    void OnStatusChange(uint8_t slotId, const uint8_t* data, uint16_t length,
+                        uint32_t completionCode) {
+        if (slotId == 0 || slotId > Xhci::MAX_SLOTS) return;
+
+        if (Succeeded(completionCode)) {
+            uint32_t bits = 0;
+            for (uint16_t i = 0; i < length && i < 4; i++) {
+                bits |= (uint32_t)data[i] << (8 * i);
+            }
+            g_changedPorts[slotId].fetch_or(bits, std::memory_order_relaxed);
+            g_rearm[slotId].store(true, std::memory_order_relaxed);
+        } else {
+            // The endpoint halted; it needs a reset before it reports again,
+            // and changes may have been missed meanwhile.
+            g_endpointFailed[slotId].store(true, std::memory_order_relaxed);
+        }
+        g_pending.store(true, std::memory_order_release);
+    }
+
+    void RequestPortCheck(uint8_t hubSlot, uint8_t port) {
+        if (hubSlot == 0 || hubSlot > Xhci::MAX_SLOTS || port == 0 || port > 31) return;
+        g_changedPorts[hubSlot].fetch_or(1u << port, std::memory_order_relaxed);
+        g_pending.store(true, std::memory_order_release);
+    }
+
+    bool HasPendingWork() {
+        return g_pending.load(std::memory_order_acquire);
+    }
+
+    void ProcessPendingWork() {
+        if (!g_pending.exchange(false, std::memory_order_acq_rel)) return;
+
+        for (uint8_t slot = 1; slot <= Xhci::MAX_SLOTS; slot++) {
+            if (!g_hubs[slot].Active) continue;
+
+            bool failed = g_endpointFailed[slot].exchange(false, std::memory_order_acq_rel);
+            bool rearm = g_rearm[slot].exchange(false, std::memory_order_acq_rel);
+            uint32_t changed = g_changedPorts[slot].exchange(0, std::memory_order_acq_rel);
+            if (!failed && !rearm && changed == 0) continue;
+
+            uint8_t ports = g_hubs[slot].Info.Ports;
+            if (ports > 31) ports = 31;
+
+            if (failed) {
+                // Reset Endpoint re-arms the endpoint itself.
+                Xhci::ResetInterruptEndpoint(slot);
+                changed = ((ports == 31) ? 0xFFFFFFFFu : ((1u << (ports + 1)) - 1)) & ~1u;
+            }
+
+            if (changed & 1u) ClearHubChanges(slot);
+            for (uint8_t port = 1; port <= ports && g_hubs[slot].Active; port++) {
+                if (changed & (1u << port)) HandlePort(slot, port, false);
+            }
+
+            if (rearm && !failed && g_hubs[slot].Active) Xhci::QueueInterruptTransfer(slot);
+        }
+    }
+
+}

kernel/src/Drivers/USB/Hub.hpp

@@ -0,0 +1,57 @@
+/*
+    * Hub.hpp
+    * USB hub class driver (USB 2.0 and SuperSpeed hubs)
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// Devices behind a hub get their own xHCI slots, addressed by route string
+// (Xhci::UsbDeviceInfo). The hub driver powers and resets the hub's ports
+// and enumerates what it finds; the hub's interrupt endpoint reports port
+// changes afterwards, handled from the xHCI deferred-work pass.
+//
+// The Raspberry Pi 4/400 needs this for any USB 2 device: the VL805's only
+// USB 2 root port carries an on-board VIA hub that every USB 2 line (the
+// Pi 400's built-in keyboard included) hangs off.
+
+namespace Drivers::USB::Hub {
+
+    struct HubDescriptorInfo {
+        uint8_t  Ports;
+        uint8_t  ThinkTime;     // TT think time code, wHubCharacteristics[6:5]
+        uint16_t PowerOnMs;     // bPwrOn2PwrGood, in ms
+        bool     SuperSpeed;
+    };
+
+    // Read the hub class descriptor of a configured hub. `speed` is the
+    // hub's xHCI port speed (SuperSpeed hubs have their own descriptor).
+    bool ReadDescriptor(uint8_t slotId, uint32_t speed, HubDescriptorInfo& out);
+
+    // Take over a hub whose slot is configured as a hub with its interrupt
+    // endpoint: power its ports, enumerate the devices on them, then arm the
+    // status-change endpoint. Process context.
+    void RegisterHub(uint8_t slotId, const HubDescriptorInfo& info);
+
+    // Forget a hub (its children are removed by Xhci::RemoveDevice).
+    void UnregisterHub(uint8_t slotId);
+
+    // A completion on the hub's status-change endpoint. Called from the
+    // event ring drain: records the change and nothing else.
+    void OnStatusChange(uint8_t slotId, const uint8_t* data, uint16_t length,
+                        uint32_t completionCode);
+
+    // Look at a port again on the next ProcessPendingWork as if it had
+    // reported a change: a device removed from it is reset and enumerated
+    // afresh, as if replugged.
+    void RequestPortCheck(uint8_t hubSlot, uint8_t port);
+
+    // Port changes are waiting for ProcessPendingWork.
+    bool HasPendingWork();
+
+    // Handle recorded port changes and re-arm the endpoints. Runs inside the
+    // xHCI deferred-work pass (process/idle context, single runner).
+    void ProcessPendingWork();
+
+}

kernel/src/Drivers/USB/UsbDevice.cpp

@@ -10,6 +10,7 @@
 #include "HidKeyboard.hpp"
 #include "HidMouse.hpp"
 #include "MassStorage.hpp"
+#include "Hub.hpp"
 #include "Bluetooth/Bluetooth.hpp"
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
@@ -65,24 +66,28 @@ namespace Drivers::USB::UsbDevice {
         return speed;  // Same encoding
     }
 
-    // Convert USB endpoint bInterval to xHCI Endpoint Context Interval value.
-    // HS/SS: bInterval is already in 2^(n-1) * 125µs encoding — use directly.
-    // FS/LS: bInterval is in milliseconds (frames) — convert via fls(bInterval * 8).
+    // Convert USB endpoint bInterval to xHCI Endpoint Context Interval value
+    // (a period of 2^Interval * 125 us), as Linux's xhci_get_endpoint_interval.
+    // HS/SS: bInterval encodes a period of 2^(bInterval-1) microframes.
+    // FS/LS: bInterval is in milliseconds (frames); round down to a power of
+    // two microframes, between 1 ms and 128 ms.
     static uint32_t ConvertInterval(uint32_t speed, uint8_t bInterval) {
-        if (bInterval == 0) return 0;
-
         if (speed == Xhci::SPEED_HIGH || speed == Xhci::SPEED_SUPER) {
-            return bInterval;
+            uint32_t b = bInterval;
+            if (b < 1) b = 1;
+            if (b > 16) b = 16;
+            return b - 1;
         }
 
-        // FS/LS: bInterval ms → microframes, then find highest set bit position
+        if (bInterval == 0) return 3;
         uint32_t microframes = (uint32_t)bInterval * 8;
         uint32_t interval = 0;
-        while (microframes > 0) {
+        while (microframes > 1) {
             interval++;
             microframes >>= 1;
         }
-        if (interval > 15) interval = 15;
+        if (interval < 3) interval = 3;
+        if (interval > 10) interval = 10;
         return interval;
     }
 
@@ -101,9 +106,25 @@ namespace Drivers::USB::UsbDevice {
     // EnumerateDevice
     // ---------------------------------------------------------------------------
 
+    static bool EnumerateInSlot(uint8_t slotId, const DeviceLocation& location, uint32_t speed,
+                                bool& isHub, Hub::HubDescriptorInfo& hubInfo);
+
     uint8_t EnumerateDevice(uint8_t portId, uint32_t speed) {
-        KernelLogStream(INFO, "USB") << "Enumerating device on port "
-            << (uint64_t)portId << " speed=" << SpeedToString(speed);
+        DeviceLocation location = {};
+        location.RootPort = portId;
+        return EnumerateDevice(location, speed);
+    }
+
+    uint8_t EnumerateDevice(const DeviceLocation& location, uint32_t speed) {
+        if (location.ParentSlot == 0) {
+            KernelLogStream(INFO, "USB") << "Enumerating device on port "
+                << (uint64_t)location.RootPort << " speed=" << SpeedToString(speed);
+        } else {
+            KernelLogStream(INFO, "USB") << "Enumerating device on hub slot "
+                << (uint64_t)location.ParentSlot << " port " << (uint64_t)location.ParentPort
+                << " (route " << base::hex << (uint64_t)location.RouteString << base::dec
+                << ", root port " << (uint64_t)location.RootPort << ") speed=" << SpeedToString(speed);
+        }
 
         // -----------------------------------------------------------------
         // Step 1: Enable Slot
@@ -125,6 +146,31 @@ namespace Drivers::USB::UsbDevice {
 
         KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId << " enabled";
 
+        bool isHub = false;
+        Hub::HubDescriptorInfo hubInfo = {};
+        if (!EnumerateInSlot(slotId, location, speed, isHub, hubInfo)) {
+            // Give the slot back, or every failed attempt (a flaky device
+            // re-reported by its hub) would leak one until none are left.
+            Xhci::TRB disableTrb = {};
+            disableTrb.Control = (Xhci::TRB_DISABLE_SLOT << Xhci::TRB_TYPE_SHIFT)
+                               | ((uint32_t)slotId << 24);
+            Xhci::SendCommand(disableTrb);
+            Xhci::g_dcbaa[slotId] = 0;
+            *Xhci::GetDevice(slotId) = {};
+            return 0;
+        }
+
+        // A hub enumerates what is behind it now that its own slot is
+        // published (its children name it as their parent).
+        if (isHub) Hub::RegisterHub(slotId, hubInfo);
+        return slotId;
+    }
+
+    static bool EnumerateInSlot(uint8_t slotId, const DeviceLocation& location, uint32_t speed,
+                                bool& isHub, Hub::HubDescriptorInfo& hubInfo) {
+        uint8_t portId = location.RootPort;
+        uint32_t cc = 0;
+
         // -----------------------------------------------------------------
         // Step 2: Allocate device output context and set DCBAA entry
         // -----------------------------------------------------------------
@@ -133,9 +179,16 @@ namespace Drivers::USB::UsbDevice {
         dev->Active = true;
         dev->PortId = portId;
         dev->Speed  = speed;
+        dev->ParentSlot  = location.ParentSlot;
+        dev->ParentPort  = location.ParentPort;
+        dev->Depth       = location.Depth;
+        dev->RouteString = location.RouteString;
+        dev->TtHubSlot   = location.TtHubSlot;
+        dev->TtPort      = location.TtPort;
+        dev->TtMulti     = location.TtMulti;
 
         // Allocate a zeroed page for the output DeviceContext
-        auto* outputCtx = (Xhci::DeviceContext*)Memory::g_pfa->AllocateZeroed();
+        auto* outputCtx = (Xhci::DeviceContext*)Xhci::AllocateDmaPage();
         dev->OutputContext     = outputCtx;
         dev->OutputContextPhys = Memory::SubHHDM(outputCtx);
 
@@ -145,7 +198,7 @@ namespace Drivers::USB::UsbDevice {
         // -----------------------------------------------------------------
         // Step 3: Build Input Context for Address Device command
         // -----------------------------------------------------------------
-        auto* inputCtx = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
+        auto* inputCtx = (Xhci::InputContext*)Xhci::AllocateDmaPage();
 
         // Input Control Context: add Slot Context (bit 0) and EP0 (bit 1)
         inputCtx->ICC.AddFlags = 0x3;
@@ -153,12 +206,18 @@ namespace Drivers::USB::UsbDevice {
         // Slot Context
         uint32_t speedVal    = SpeedToSlotContextValue(speed);
         uint32_t ctxEntries  = 1;  // Context Entries = 1 (Slot + EP0 only)
-        inputCtx->Slot.Field0 = (ctxEntries << 27) | (speedVal << 20);
+        inputCtx->Slot.Field0 = (ctxEntries << 27) | (speedVal << 20)
+                              | (location.RouteString & 0xFFFFF)
+                              | (location.TtMulti ? Xhci::SLOT_MTT : 0);
         inputCtx->Slot.Field1 = ((uint32_t)portId << 16);  // Root Hub Port Number
+        // Low/full-speed behind a high-speed hub: the hub slot and port
+        // whose transaction translator the controller addresses.
+        inputCtx->Slot.Field2 = (uint32_t)location.TtHubSlot
+                              | ((uint32_t)location.TtPort << 8);
 
         // EP0 Context
         // Allocate EP0 transfer ring
-        auto* ep0Ring = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+        auto* ep0Ring = (Xhci::TRB*)Xhci::AllocateDmaPage();
         dev->EP0Ring        = ep0Ring;
         dev->EP0RingPhys    = Memory::SubHHDM(ep0Ring);
         dev->EP0RingEnqueue = 0;
@@ -198,8 +257,7 @@ namespace Drivers::USB::UsbDevice {
         if (cc != Xhci::CC_SUCCESS) {
             KernelLogStream(ERROR, "USB") << "Address Device (BSR=1) failed, slot="
                 << (uint64_t)slotId << " cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId << " initialized (BSR=1)";
@@ -213,8 +271,7 @@ namespace Drivers::USB::UsbDevice {
                                    partialDesc, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(8-byte) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         uint8_t bMaxPacketSize0 = partialDesc[7];
@@ -227,7 +284,7 @@ namespace Drivers::USB::UsbDevice {
         // Step 4c: Evaluate Context — update EP0 max packet size if needed
         // -----------------------------------------------------------------
         if (bMaxPacketSize0 != maxPacket) {
-            auto* evalCtx = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
+            auto* evalCtx = (Xhci::InputContext*)Xhci::AllocateDmaPage();
 
             // Only updating EP0 — set AddFlags bit 1 (EP0), no slot context needed
             evalCtx->ICC.AddFlags = (1 << 1);
@@ -280,8 +337,7 @@ namespace Drivers::USB::UsbDevice {
         if (cc != Xhci::CC_SUCCESS) {
             KernelLogStream(ERROR, "USB") << "Address Device failed, slot="
                 << (uint64_t)slotId << " cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         // Set-address recovery time (USB spec requires >= 2ms, use 10ms for safety)
@@ -298,8 +354,7 @@ namespace Drivers::USB::UsbDevice {
                                    &devDesc, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Device) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         dev->VendorId    = devDesc.idVendor;
@@ -319,8 +374,7 @@ namespace Drivers::USB::UsbDevice {
                                    &cfgHdr, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Config header) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         uint8_t cfgBuf[256] = {};
@@ -332,8 +386,7 @@ namespace Drivers::USB::UsbDevice {
                                    cfgBuf, true);
         if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
             KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Config full) failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
         }
 
         // -----------------------------------------------------------------
@@ -341,6 +394,7 @@ namespace Drivers::USB::UsbDevice {
         // -----------------------------------------------------------------
         uint16_t offset = 0;
         bool foundHid = false;
+        bool currentHub = false;
         bool foundBt = false;
         bool foundMsc = false;
         bool currentMsc = false;
@@ -358,9 +412,23 @@ namespace Drivers::USB::UsbDevice {
                 auto* iface = (InterfaceDescriptor*)&cfgBuf[offset];
                 // Reset at each new interface boundary
                 foundHid = false;
+                currentHub = false;
                 foundBt = false;
                 currentMsc = false;
 
+                // Hub (class 9): its status-change interrupt endpoint. Only
+                // alternate setting 0 (single-TT) is used.
+                if (!foundEp && !isHub &&
+                    iface->bInterfaceClass == CLASS_HUB &&
+                    iface->bAlternateSetting == 0) {
+                    dev->InterfaceClass    = iface->bInterfaceClass;
+                    dev->InterfaceSubClass = iface->bInterfaceSubClass;
+                    dev->InterfaceProtocol = iface->bInterfaceProtocol;
+                    dev->InterfaceNumber   = iface->bInterfaceNumber;
+                    currentHub = true;
+                    isHub = true;
+                }
+
                 if (!foundEp &&
                     iface->bInterfaceClass == CLASS_HID &&
                     iface->bInterfaceSubClass == SUBCLASS_BOOT) {
@@ -408,8 +476,8 @@ namespace Drivers::USB::UsbDevice {
                 uint8_t xferType = ep->bmAttributes & EP_XFER_TYPE_MASK;
                 bool isIn = (ep->bEndpointAddress & EP_DIR_IN) != 0;
 
-                // HID interrupt IN endpoint
-                if (foundHid && !foundEp && isIn && xferType == EP_XFER_INTERRUPT) {
+                // HID or hub interrupt IN endpoint
+                if ((foundHid || currentHub) && !foundEp && isIn && xferType == EP_XFER_INTERRUPT) {
                     dev->InterruptEpNum     = ep->bEndpointAddress & 0x0F;
                     dev->InterruptMaxPacket = ep->wMaxPacketSize & 0x7FF;
                     dev->InterruptInterval  = ep->bInterval;
@@ -455,7 +523,7 @@ namespace Drivers::USB::UsbDevice {
         // The xHCI slot model currently stores one interface; a future model
         // can retain every alternate/interface without changing the userspace
         // claim ABI, which already names the interface number explicitly.
-        bool knownInterface = foundBt || foundMsc ||
+        bool knownInterface = foundBt || foundMsc || isHub ||
             dev->InterfaceClass == CLASS_HID;
         if (!knownInterface) {
             bool inFirstInterface = false;
@@ -502,15 +570,23 @@ namespace Drivers::USB::UsbDevice {
                                    cfgHdr.bConfigurationValue, 0, 0, nullptr, false);
         if (cc != Xhci::CC_SUCCESS) {
             KernelLogStream(ERROR, "USB") << "SET_CONFIGURATION failed, cc=" << (uint64_t)cc;
-            dev->Active = false;
-            return 0;
+            return false;
+        }
+
+        // -----------------------------------------------------------------
+        // Step 8b: Hub descriptor (the slot context must describe the hub)
+        // -----------------------------------------------------------------
+        if (isHub && (!foundEp || !Hub::ReadDescriptor(slotId, speed, hubInfo))) {
+            KernelLogStream(WARNING, "USB") << "Slot " << (uint64_t)slotId
+                << ": hub left unconfigured";
+            isHub = false;
         }
 
         // -----------------------------------------------------------------
         // Step 9: Configure Endpoints
         // -----------------------------------------------------------------
         if (foundEp || foundBulkIn || foundBulkOut) {
-            auto* inputCtx2 = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
+            auto* inputCtx2 = (Xhci::InputContext*)Xhci::AllocateDmaPage();
 
             // Start with slot context
             inputCtx2->ICC.AddFlags = (1 << 0);
@@ -528,7 +604,7 @@ namespace Drivers::USB::UsbDevice {
                 if (dci > maxDci) maxDci = dci;
 
                 // Allocate interrupt transfer ring
-                auto* intRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+                auto* intRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
                 dev->InterruptRing        = intRing;
                 dev->InterruptRingPhys    = Memory::SubHHDM(intRing);
                 dev->InterruptRingEnqueue = 0;
@@ -542,6 +618,10 @@ namespace Drivers::USB::UsbDevice {
 
                 auto& epCtx = inputCtx2->EP[dci - 1];
                 uint32_t xhciInterval = ConvertInterval(speed, dev->InterruptInterval);
+                // Split transactions through a hub's TT: poll at least every
+                // 8 ms. Cheap, and conservative on the VL805, which Linux
+                // gives a quirk for periodic endpoints behind a TT.
+                if (dev->TtHubSlot != 0 && xhciInterval > 6) xhciInterval = 6;
                 epCtx.Field0 = (xhciInterval << 16);
                 epCtx.Field1 = (3 << 1)
                              | (Xhci::EP_TYPE_INTERRUPT_IN << 3)
@@ -556,7 +636,7 @@ namespace Drivers::USB::UsbDevice {
                 inputCtx2->ICC.AddFlags |= (1 << dci);
                 if (dci > maxDci) maxDci = dci;
 
-                auto* bulkInRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+                auto* bulkInRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
                 dev->BulkInRing        = bulkInRing;
                 dev->BulkInRingPhys    = Memory::SubHHDM(bulkInRing);
                 dev->BulkInRingEnqueue = 0;
@@ -583,7 +663,7 @@ namespace Drivers::USB::UsbDevice {
                 inputCtx2->ICC.AddFlags |= (1 << dci);
                 if (dci > maxDci) maxDci = dci;
 
-                auto* bulkOutRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
+                auto* bulkOutRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
                 dev->BulkOutRing        = bulkOutRing;
                 dev->BulkOutRingPhys    = Memory::SubHHDM(bulkOutRing);
                 dev->BulkOutRingEnqueue = 0;
@@ -608,6 +688,19 @@ namespace Drivers::USB::UsbDevice {
             inputCtx2->Slot.Field0 = (inputCtx2->Slot.Field0 & ~(0x1Fu << 27))
                                    | (maxDci << 27);
 
+            // A hub: Hub flag, Number of Ports and, for a high-speed hub, its
+            // TT Think Time (xHCI 6.2.2). The controller needs them to
+            // schedule split transactions for what sits behind it.
+            if (isHub) {
+                inputCtx2->Slot.Field0 |= Xhci::SLOT_HUB;
+                inputCtx2->Slot.Field1 = (inputCtx2->Slot.Field1 & 0x00FFFFFF)
+                                       | ((uint32_t)hubInfo.Ports << 24);
+                if (speed == Xhci::SPEED_HIGH) {
+                    inputCtx2->Slot.Field2 = (inputCtx2->Slot.Field2 & ~(3u << 16))
+                                           | ((uint32_t)hubInfo.ThinkTime << 16);
+                }
+            }
+
             // Send Configure Endpoint command
             Xhci::TRB cfgTrb = {};
             uint64_t inputCtx2Phys = Memory::SubHHDM(inputCtx2);
@@ -620,8 +713,7 @@ namespace Drivers::USB::UsbDevice {
             if (cc != Xhci::CC_SUCCESS) {
                 KernelLogStream(ERROR, "USB") << "Configure Endpoint failed, slot="
                     << (uint64_t)slotId << " cc=" << (uint64_t)cc;
-                dev->Active = false;
-                return 0;
+                return false;
             }
 
             if (foundEp) {
@@ -713,6 +805,8 @@ namespace Drivers::USB::UsbDevice {
             KernelLogStream(OK, "USB") << "Slot " << (uint64_t)slotId << ": Bluetooth Adapter"
                 << " VID:" << base::hex << (uint64_t)dev->VendorId
                 << " PID:" << (uint64_t)dev->ProductId << base::dec;
+        } else if (isHub) {
+            dev->KernelDriverBound = true;
         } else if (dev->InterfaceClass == CLASS_MASS_STORAGE &&
                    dev->InterfaceSubClass == SUBCLASS_SCSI &&
                    dev->InterfaceProtocol == PROTOCOL_BULK_ONLY &&
@@ -733,7 +827,7 @@ namespace Drivers::USB::UsbDevice {
         // Publish to userspace only after endpoint configuration and kernel
         // class-driver binding decisions are complete.
         dev->Ready = true;
-        return slotId;
+        return true;
     }
 
 }

kernel/src/Drivers/USB/UsbDevice.hpp

@@ -80,6 +80,9 @@ namespace Drivers::USB::UsbDevice {
     constexpr uint8_t PROTOCOL_KEYBOARD  = 0x01;
     constexpr uint8_t PROTOCOL_MOUSE     = 0x02;
 
+    // Hub class
+    constexpr uint8_t CLASS_HUB          = 0x09;
+
     // Wireless Controller class (Bluetooth)
     constexpr uint8_t CLASS_WIRELESS     = 0xE0;
     constexpr uint8_t SUBCLASS_RF        = 0x01;
@@ -119,8 +122,24 @@ namespace Drivers::USB::UsbDevice {
     // Public API
     // ---------------------------------------------------------------------------
 
-    // Enumerate a newly connected device on the given port with the given speed.
+    // Where a device is attached: a root hub port, or a port of a hub
+    // (Xhci::UsbDeviceInfo describes the fields).
+    struct DeviceLocation {
+        uint8_t  RootPort;
+        uint8_t  ParentSlot;
+        uint8_t  ParentPort;
+        uint8_t  Depth;
+        uint32_t RouteString;
+        uint8_t  TtHubSlot;
+        uint8_t  TtPort;
+        bool     TtMulti;
+    };
+
+    // Enumerate a newly connected device on the given root port with the given speed.
     // Returns the assigned slot ID, or 0 on failure.
     uint8_t EnumerateDevice(uint8_t portId, uint32_t speed);
 
+    // Enumerate a device at `location` (a hub port that has been reset).
+    uint8_t EnumerateDevice(const DeviceLocation& location, uint32_t speed);
+
 };

kernel/src/Drivers/USB/Xhci.cpp

@@ -11,11 +11,14 @@
 #include "HidKeyboard.hpp"
 #include "HidMouse.hpp"
 #include "MassStorage.hpp"
+#include "Hub.hpp"
 #include "UserUsb.hpp"
 #include <Pci/Pci.hpp>
 #include <Terminal/Terminal.hpp>
 #include <CppLib/Stream.hpp>
 #include <Memory/HHDM.hpp>
+#include <Memory/Dma.hpp>
+#include <ACPI/GsiRouting.hpp>
 #include <Memory/Paging.hpp>
 #include <Memory/PageFrameAllocator.hpp>
 #include <Libraries/Memory.hpp>
@@ -59,6 +62,21 @@ namespace Drivers::USB::Xhci {
     static std::atomic<bool> g_hotplugProcessing{false};
     static std::atomic<bool> g_irqDrainPending{false};
 
+    // HID interrupt endpoints that completed with an error. The event drain
+    // does not re-arm them (a halted endpoint takes commands to clear, which
+    // cannot be waited on from there): it records the error, and the
+    // deferred-work pass recovers the endpoint once g_hidRecoverDueMs is
+    // reached. Low/full-speed devices behind a hub's TT (every mouse on a
+    // Raspberry Pi 4) see transaction and split errors often enough that
+    // without this the device goes dead until replugged.
+    // g_hidErrorStreak counts errors since the last good report; it sets the
+    // backoff, and at HID_REENUMERATE_STREAK the device is reset and
+    // enumerated again, as a replug would.
+    static constexpr uint32_t HID_REENUMERATE_STREAK = 6;
+    static std::atomic<uint32_t> g_hidErrorCc[MAX_SLOTS + 1] = {};       // 0 = no recovery due
+    static std::atomic<uint32_t> g_hidErrorStreak[MAX_SLOTS + 1] = {};
+    static std::atomic<uint64_t> g_hidRecoverDueMs[MAX_SLOTS + 1] = {};
+
     // PollEvents also runs directly from the MSI handler. KernelLogStream uses
     // a process-context mutex, so printing there can deadlock if the interrupt
     // preempts a CPU that already owns the terminal lock. Capture the small
@@ -243,18 +261,44 @@ namespace Drivers::USB::Xhci {
         return *(volatile uint32_t*)(g_mmioBase + reg);
     }
 
+    // TRBs (and the buffers they point at) must be visible before the
+    // controller is told to look.
     static void WriteDoorbell(uint32_t index, uint32_t value) {
+        Arch::DmaWriteFence();
         *(volatile uint32_t*)(g_dbBase + index * 4) = value;
     }
 
+    // Hand a TRB to the controller: everything else in it first, then the
+    // control word that carries its cycle bit. A running ring may fetch the
+    // TRB the moment the cycle bit matches.
+    static inline void PublishTrb(TRB& trb, uint32_t control) {
+        Arch::DmaWriteFence();
+        *(volatile uint32_t*)&trb.Control = control;
+    }
+
     // -------------------------------------------------------------------------
     // DMA buffer allocation
     // -------------------------------------------------------------------------
 
+    // Whether the controller's DMA snoops the CPU caches. PCI controllers
+    // are taken to be coherent; a platform controller says so in _CCA.
+    static bool g_dmaCoherent = true;
+
+    uint8_t* AllocateDmaPage(uint64_t& outPhysAddr) {
+        return (uint8_t*)Memory::Dma::AllocatePage(outPhysAddr, g_dmaCoherent);
+    }
+
+    void* AllocateDmaPage() {
+        uint64_t phys;
+        return AllocateDmaPage(phys);
+    }
+
+    void FreeDmaPage(void* page) {
+        Memory::Dma::FreePage(page, g_dmaCoherent);
+    }
+
     static uint8_t* AllocateDmaBuffer(uint64_t& outPhysAddr) {
-        void* virt = Memory::g_pfa->AllocateZeroed();
-        outPhysAddr = Memory::SubHHDM(virt);
-        return (uint8_t*)virt;
+        return AllocateDmaPage(outPhysAddr);
     }
 
     // -------------------------------------------------------------------------
@@ -419,6 +463,24 @@ namespace Drivers::USB::Xhci {
     // MSI setup (same pattern as E1000E)
     // -------------------------------------------------------------------------
 
+    // Legacy INTx, for controllers without MSI (or platforms that cannot
+    // receive it): the line the firmware or Pci::AssignInterruptLines put
+    // in the Interrupt Line register. HandleInterrupt suits a level line:
+    // clearing IMAN.IP drops it.
+    static bool SetupIntx(uint8_t bus, uint8_t dev, uint8_t func) {
+        uint8_t line = Pci::LegacyRead8(bus, dev, func, (uint8_t)Pci::PCI_REG_INTERRUPT);
+        if (line == 0 || line >= 24) return false;
+
+        uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
+        Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND,
+                           pciCmd & ~Pci::PCI_CMD_INTX_DISABLE);
+        Arch::RegisterIrqHandler(line, HandleInterrupt);
+        Arch::EnableLegacyIrq(line);
+
+        KernelLogStream(OK, "xHCI") << "Using INTx: platform line " << base::dec << (uint64_t)line;
+        return true;
+    }
+
     static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
         uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
         if (cap == 0) {
@@ -523,6 +585,8 @@ namespace Drivers::USB::Xhci {
             if (evtCycle != g_evtRingCCS) {
                 break; // No more events
             }
+            // The rest of the event was written before its cycle bit.
+            Arch::DmaReadFence();
 
             uint32_t trbType = (evt.Control & TRB_TYPE_MASK) >> TRB_TYPE_SHIFT;
             lastType = trbType;
@@ -545,8 +609,12 @@ namespace Drivers::USB::Xhci {
                     WriteOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE,
                             (portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
 
-                    // Defer enumeration to ProcessDeferredWork outside interrupt context.
-                    if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
+                    // Defer enumeration to ProcessDeferredWork outside interrupt
+                    // context. Changes during the boot scan are kept too (that
+                    // pass skips ports already enumerated): a device that
+                    // connects after the scan looked at its port, slower on
+                    // real hardware than in QEMU, would otherwise be lost.
+                    if (portId >= 1 && portId <= g_maxPorts) {
                         g_hotplugPending[portId - 1].store(true, std::memory_order_release);
                         g_deferredWorkPending.store(true, std::memory_order_release);
                     }
@@ -642,12 +710,17 @@ namespace Drivers::USB::Xhci {
                                 uint16_t len = (residual < reqLen)
                                     ? (uint16_t)(reqLen - residual) : 0;
 
-                                if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
+                                if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
+                                    // Re-armed once the change is handled.
+                                    Hub::OnStatusChange((uint8_t)slotId, g_interruptDataBuf[slotId],
+                                                        len, completionCode);
+                                } else if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
                                     if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
                                         HidKeyboard::ProcessReport(g_interruptDataBuf[slotId], len);
                                     } else if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
                                         HidMouse::ProcessReport(g_interruptDataBuf[slotId], len);
                                     }
+                                    g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
                                     // Re-queue for next HID report
                                     QueueInterruptTransfer(slotId);
                                 } else if (g_transferCallbacks[slotId]) {
@@ -672,6 +745,23 @@ namespace Drivers::USB::Xhci {
                                 g_diagnosticWorkPending.store(true, std::memory_order_release);
                             }
 
+                            if (dev.InterfaceClass == UsbDevice::CLASS_HUB && dev.InterruptEpNum &&
+                                epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
+                                Hub::OnStatusChange((uint8_t)slotId, nullptr, 0, completionCode);
+                            }
+
+                            if (dev.InterfaceClass == UsbDevice::CLASS_HID && dev.InterruptEpNum &&
+                                epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
+                                // Retry at once, then back off 10 ms doubling.
+                                uint32_t streak = g_hidErrorStreak[slotId].fetch_add(
+                                    1, std::memory_order_relaxed) + 1;
+                                uint64_t delay = streak <= 1 ? 0 : 10ull << (streak - 2);
+                                if (delay > 1000) delay = 1000;
+                                g_hidRecoverDueMs[slotId].store(
+                                    Timekeeping::GetMilliseconds() + delay, std::memory_order_relaxed);
+                                g_hidErrorCc[slotId].store(completionCode, std::memory_order_release);
+                            }
+
                             // Notify callback of errors too
                             if (g_transferCallbacks[slotId]) {
                                 g_transferCallbacks[slotId](slotId, epDci,
@@ -787,7 +877,7 @@ namespace Drivers::USB::Xhci {
         if (g_cmdRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        slot.Control = control;
+        PublishTrb(slot, control);
 
         // Advance enqueue pointer
         g_cmdRingEnqueue++;
@@ -836,10 +926,21 @@ namespace Drivers::USB::Xhci {
     // ControlTransfer - perform a control transfer on EP0
     // -------------------------------------------------------------------------
 
+    static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
+                                          uint16_t wValue, uint16_t wIndex, uint16_t wLength,
+                                          void* data, uint64_t dataPhys, bool dirIn, bool trueNested);
+
+    // Data stages of a non-coherent controller go through this page (taken
+    // under g_controlXferLock). Callers pass small stack buffers, and cache
+    // maintenance cannot make those safe: a cache line the buffer shares with
+    // live stack data that the CPU dirties while the transfer runs is written
+    // back over what the device wrote.
+    static uint8_t* g_ctrlBounce = nullptr;
+    static uint64_t g_ctrlBouncePhys = 0;
+
     uint32_t ControlTransfer(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
                              uint16_t wValue, uint16_t wIndex, uint16_t wLength,
                              void* data, bool dirIn) {
-
         if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
             return 0xFF;
         }
@@ -853,6 +954,32 @@ namespace Drivers::USB::Xhci {
                           g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
         if (!trueNested) g_controlXferLock.Acquire();
 
+        bool hasData = data != nullptr && wLength != 0;
+        bool bounce = hasData && !g_dmaCoherent && !trueNested &&
+                      g_ctrlBounce != nullptr && wLength <= 0x1000;
+        uint32_t cc;
+        if (bounce) {
+            // Copied in both directions: bytes a short IN transfer leaves
+            // alone keep the caller's contents, as with a direct transfer.
+            memcpy(g_ctrlBounce, data, wLength);
+            cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex,
+                                       wLength, g_ctrlBounce, g_ctrlBouncePhys, dirIn, false);
+            if (dirIn) memcpy(data, g_ctrlBounce, wLength);
+        } else {
+            // The data stage buffer is the caller's (cacheable) memory.
+            if (hasData) Memory::Dma::SyncForDevice(data, wLength, g_dmaCoherent);
+            cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex, wLength,
+                                       data, hasData ? Memory::SubHHDM(data) : 0, dirIn, trueNested);
+            if (hasData && dirIn) Memory::Dma::SyncForCpu(data, wLength, g_dmaCoherent);
+        }
+
+        if (!trueNested) g_controlXferLock.Release();
+        return cc;
+    }
+
+    static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
+                                          uint16_t wValue, uint16_t wIndex, uint16_t wLength,
+                                          void* data, uint64_t dataPhys, bool dirIn, bool trueNested) {
         UsbDeviceInfo& dev = g_devices[slotId];
 
         // --- Setup Stage TRB ---
@@ -870,15 +997,15 @@ namespace Drivers::USB::Xhci {
         if (dev.EP0RingCCS) {
             setupControl |= TRB_CYCLE_BIT;
         }
-        setup.Control = setupControl;
+        // The Setup TRB's cycle bit is published only after the Data and
+        // Status TRBs are complete (below): if EP0 is still running, it
+        // would otherwise run into a half-built TD.
 
         // Advance EP0 enqueue (handles Link TRB wrap)
         AdvanceEP0Ring(dev);
 
         // --- Data Stage TRB (if wLength > 0) ---
         if (wLength > 0 && data != nullptr) {
-            uint64_t dataPhys = Memory::SubHHDM(data);
-
             TRB& dataTrb = dev.EP0Ring[dev.EP0RingEnqueue];
             dataTrb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
             dataTrb.Parameter1 = (uint32_t)(dataPhys >> 32);
@@ -891,7 +1018,7 @@ namespace Drivers::USB::Xhci {
             if (dev.EP0RingCCS) {
                 dataControl |= TRB_CYCLE_BIT;
             }
-            dataTrb.Control = dataControl;
+            PublishTrb(dataTrb, dataControl);
 
             AdvanceEP0Ring(dev);
         }
@@ -912,10 +1039,12 @@ namespace Drivers::USB::Xhci {
         if (dev.EP0RingCCS) {
             statusControl |= TRB_CYCLE_BIT;
         }
-        status.Control = statusControl;
+        PublishTrb(status, statusControl);
 
         AdvanceEP0Ring(dev);
 
+        PublishTrb(setup, setupControl);
+
         // Ring doorbell for this slot, target EP0 (DCI 1)
         g_xferCompleted[slotId] = false;
         WriteDoorbell(slotId, 1);
@@ -939,7 +1068,6 @@ namespace Drivers::USB::Xhci {
         while (Timekeeping::GetMilliseconds() - xferStart < 2000) {
             PollEvents();
             if (g_xferCompleted[slotId]) {
-                g_controlXferLock.Release();
                 return g_xferCompletionCode[slotId];
             }
             for (int j = 0; j < 100; j++) {
@@ -973,7 +1101,6 @@ namespace Drivers::USB::Xhci {
                           | (1 << 16);  // DCI=1 (EP0)
         SendCommand(deqTrb);
 
-        g_controlXferLock.Release();
         return 0xFF;
     }
 
@@ -1003,7 +1130,7 @@ namespace Drivers::USB::Xhci {
         if (dev.InterruptRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        trb.Control = control;
+        PublishTrb(trb, control);
 
         // Advance enqueue (handles Link TRB wrap)
         AdvanceInterruptRing(dev);
@@ -1140,7 +1267,7 @@ namespace Drivers::USB::Xhci {
         if (dev.BulkInRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        trb.Control = control;
+        PublishTrb(trb, control);
 
         AdvanceBulkInRing(dev);
 
@@ -1267,6 +1394,9 @@ namespace Drivers::USB::Xhci {
     // -------------------------------------------------------------------------
 
     void QueueBulkOutTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
+        if (data != nullptr && length != 0) {
+            Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
+        }
         if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
 
         UsbDeviceInfo& dev = g_devices[slotId];
@@ -1281,7 +1411,7 @@ namespace Drivers::USB::Xhci {
         if (dev.BulkOutRingCCS) {
             control |= TRB_CYCLE_BIT;
         }
-        trb.Control = control;
+        PublishTrb(trb, control);
 
         AdvanceBulkOutRing(dev);
 
@@ -1294,8 +1424,23 @@ namespace Drivers::USB::Xhci {
     // BulkTransfer - blocking bulk IN/OUT transfer
     // -------------------------------------------------------------------------
 
+    static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
+                                         uint32_t length, uint32_t* actualLength);
+
     uint32_t BulkTransfer(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
                           uint32_t length, uint32_t* actualLength) {
+        if (data != nullptr && length != 0) {
+            Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
+        }
+        uint32_t cc = BulkTransferUnsynced(slotId, dirIn, data, dataPhys, length, actualLength);
+        if (data != nullptr && length != 0 && dirIn) {
+            Memory::Dma::SyncForCpu(data, length, g_dmaCoherent);
+        }
+        return cc;
+    }
+
+    static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
+                                         uint32_t length, uint32_t* actualLength) {
         if (actualLength) *actualLength = 0;
         if (length == 0) return CC_SUCCESS;
         if (length > 0x10000) return 0xFF;
@@ -1318,7 +1463,7 @@ namespace Drivers::USB::Xhci {
 
             uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
             if (dev.BulkInRingCCS) control |= TRB_CYCLE_BIT;
-            trb.Control = control;
+            PublishTrb(trb, control);
 
             AdvanceBulkInRing(dev);
             target = dev.BulkInEpNum * 2 + 1;
@@ -1332,7 +1477,7 @@ namespace Drivers::USB::Xhci {
 
             uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
             if (dev.BulkOutRingCCS) control |= TRB_CYCLE_BIT;
-            trb.Control = control;
+            PublishTrb(trb, control);
 
             AdvanceBulkOutRing(dev);
             target = dev.BulkOutEpNum * 2;
@@ -1412,6 +1557,17 @@ namespace Drivers::USB::Xhci {
         return g_initialized;
     }
 
+    // A HID endpoint whose error backoff has run out.
+    static bool HidRecoveryDue() {
+        uint64_t now = 0;
+        for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+            if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
+            if (now == 0) now = Timekeeping::GetMilliseconds();
+            if (now >= g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) return true;
+        }
+        return false;
+    }
+
     bool HasDeferredWork() {
         if (g_initialized && g_pollMode && EventRingHasWork()) {
             g_irqDrainPending.store(true, std::memory_order_release);
@@ -1419,6 +1575,7 @@ namespace Drivers::USB::Xhci {
         return g_initialized &&
             (g_irqDrainPending.load(std::memory_order_acquire) ||
              g_deferredWorkPending.load(std::memory_order_acquire) ||
+             (g_bootScanComplete && (Hub::HasPendingWork() || HidRecoveryDue())) ||
              g_diagnosticWorkPending.load(std::memory_order_acquire));
     }
 
@@ -1468,7 +1625,9 @@ namespace Drivers::USB::Xhci {
     }
 
     static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
-        if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
+        if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
+            Hub::UnregisterHub(slotId);
+        } else if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
             MassStorage::UnregisterDevice(slotId);
         } else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
                    dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
@@ -1491,6 +1650,105 @@ namespace Drivers::USB::Xhci {
         }
     }
 
+    void RemoveDevice(uint8_t slotId) {
+        if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
+
+        // Children first: a hub's slot outlives the devices addressed through it.
+        for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+            if (s != slotId && g_devices[s].Active && g_devices[s].ParentSlot == slotId) {
+                RemoveDevice(s);
+            }
+        }
+
+        UserUsb::DeviceDisconnected(slotId);
+        UnregisterClassDriver(slotId, g_devices[slotId]);
+        g_devices[slotId].Active = false;
+        g_hidErrorCc[slotId].store(0, std::memory_order_relaxed);
+        g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
+        g_transferCallbacks[slotId] = nullptr;
+        DisableSlot(slotId);
+        g_dcbaa[slotId] = 0;
+        g_devices[slotId] = {};
+        KernelLogStream(INFO, "xHCI") << "Slot " << base::dec << (uint64_t)slotId << " deactivated";
+    }
+
+    // Bring a HID device's interrupt endpoint back after a failed transfer.
+    // Reset Endpoint clears a halt (a Context State Error when the error did
+    // not halt it), Set TR Dequeue skips the failed TRB, and CLEAR_FEATURE
+    // (ENDPOINT_HALT) clears a stall on the device and puts its data toggle
+    // back to DATA0 to match the reset host side (USB 2.0 9.4.5). If errors keep coming, the
+    // device is removed and its port reset, as a replug would.
+    static void RecoverHidEndpoint(uint8_t slotId, uint32_t cc) {
+        UsbDeviceInfo& dev = g_devices[slotId];
+        if (!dev.Active || dev.InterfaceClass != UsbDevice::CLASS_HID ||
+            dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
+
+        uint32_t streak = g_hidErrorStreak[slotId].load(std::memory_order_relaxed);
+        if (streak >= HID_REENUMERATE_STREAK) {
+            KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
+                << ": HID endpoint keeps failing (cc=" << (uint64_t)cc
+                << "), resetting the device";
+            uint8_t parentSlot = dev.ParentSlot, parentPort = dev.ParentPort;
+            uint8_t rootPort = dev.PortId;
+            RemoveDevice(slotId);
+            if (parentSlot != 0) {
+                Hub::RequestPortCheck(parentSlot, parentPort);
+            } else if (rootPort >= 1 && rootPort <= g_maxPorts) {
+                // Disable the port (PED is write-1-to-clear) so the hot-plug
+                // pass resets it before enumerating.
+                uint32_t reg = OP_PORTSC_BASE + (rootPort - 1) * OP_PORTSC_STRIDE;
+                WriteOp(reg, (ReadOp(reg) & PORTSC_PRESERVE) | PORTSC_PED);
+                g_hotplugPending[rootPort - 1].store(true, std::memory_order_release);
+                g_deferredWorkPending.store(true, std::memory_order_release);
+            }
+            return;
+        }
+
+        if (streak <= 1) {
+            KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
+                << ": HID transfer failed (cc=" << (uint64_t)cc << "), recovering endpoint";
+        }
+
+        uint8_t dci = dev.InterruptEpNum * 2 + 1;
+
+        TRB resetTrb = {};
+        resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
+                         | ((uint32_t)slotId << 24)
+                         | ((uint32_t)dci << 16);
+        bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
+
+        uint64_t newDeq = dev.InterruptRingPhys
+                        + (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
+        if (dev.InterruptRingCCS) newDeq |= 1;  // DCS bit
+
+        TRB deqTrb = {};
+        deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
+        deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
+        deqTrb.Control    = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
+                          | ((uint32_t)slotId << 24)
+                          | ((uint32_t)dci << 16);
+        SendCommand(deqTrb);
+
+        // Only after a real reset: the host toggle is untouched otherwise.
+        if (wasHalted) {
+            ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
+                            0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
+                            (uint16_t)(0x80 | dev.InterruptEpNum), 0, nullptr, false);
+        }
+
+        if (dev.Active) QueueInterruptTransfer(slotId);
+    }
+
+    static void ProcessHidRecovery() {
+        uint64_t now = Timekeeping::GetMilliseconds();
+        for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+            if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
+            if (now < g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) continue;
+            uint32_t cc = g_hidErrorCc[s].exchange(0, std::memory_order_acq_rel);
+            if (cc != 0) RecoverHidEndpoint(s, cc);
+        }
+    }
+
     // -------------------------------------------------------------------------
     // ProcessDeferredWork - handle hot-plug outside interrupt context.
     // Called from the BSP idle path so class-driver probing can wait on timers.
@@ -1508,7 +1766,8 @@ namespace Drivers::USB::Xhci {
 
         bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
         bool haveHotplug = g_bootScanComplete &&
-            g_deferredWorkPending.load(std::memory_order_acquire);
+            (g_deferredWorkPending.load(std::memory_order_acquire) || Hub::HasPendingWork() ||
+             HidRecoveryDue());
         if (!haveDiagnostics && !haveHotplug) return;
 
         // CAS claim: any idle core may call this, but only one runs at a time.
@@ -1525,6 +1784,9 @@ namespace Drivers::USB::Xhci {
             return;
         }
 
+        // First: a device it gives up on is queued for the hot-plug pass below.
+        ProcessHidRecovery();
+
         g_deferredWorkPending.store(false, std::memory_order_release);
 
         for (uint32_t port = 0; port < g_maxPorts; port++) {
@@ -1585,25 +1847,23 @@ namespace Drivers::USB::Xhci {
                     UsbDevice::EnumerateDevice(port + 1, speed);
                 }
             } else {
-                // Device disconnected — deactivate its slot
+                // Device disconnected: deactivate its slot (and, for a hub,
+                // the slots of everything behind it)
                 for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
-                    if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
-                        UserUsb::DeviceDisconnected(s);
-                        UnregisterClassDriver(s, g_devices[s]);
-                        g_devices[s].Active = false;
-                        g_transferCallbacks[s] = nullptr;
-                        DisableSlot(s);
-                        g_dcbaa[s] = 0;
-                        g_devices[s] = {};
-                        KernelLogStream(INFO, "xHCI") << "Hot-unplug: slot "
-                            << base::dec << (uint64_t)s << " (port "
-                            << (uint64_t)(port + 1) << ") deactivated";
+                    if (g_devices[s].Active && g_devices[s].PortId == port + 1 &&
+                        g_devices[s].ParentSlot == 0) {
+                        KernelLogStream(INFO, "xHCI") << "Hot-unplug: port "
+                            << base::dec << (uint64_t)(port + 1);
+                        RemoveDevice(s);
                         break;
                     }
                 }
             }
         }
 
+        // Port changes behind hubs.
+        Hub::ProcessPendingWork();
+
         g_hotplugProcessing.store(false, std::memory_order_release);
     }
 
@@ -1611,32 +1871,22 @@ namespace Drivers::USB::Xhci {
     // Probe (called by PCI driver matching framework)
     // -------------------------------------------------------------------------
 
-    bool Probe(const Pci::PciDevice& dev) {
-        if (g_initialized) return false;
-
-        KernelLogStream(OK, "xHCI") << "Found controller at PCI "
-            << base::hex << (uint64_t)dev.Bus << ":"
-            << (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
-
-        // Read BAR0 and map MMIO region
-        uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
-
-        KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
-
-        constexpr uint64_t MmioSize = 0x10000;
-        for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
+    // Map the register block, reset the controller and set up its
+    // context and command/event rings. Interrupt delivery is the caller's.
+    static void InitializeController(uint64_t mmioPhys, uint64_t mmioSize) {
+        for (uint64_t offset = 0; offset < mmioSize; offset += 0x1000) {
             Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
         }
 
         g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
 
-        Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
-        KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
+        // Parse capability registers. CAPLENGTH and HCIVERSION share the
+        // first dword; read it whole, since narrower accesses are not safe
+        // behind every PCIe bridge (Linux does the same).
+        uint32_t capBase = ReadCap(CAP_CAPLENGTH);
+        g_capLength = (uint8_t)(capBase & 0xFF);
 
-        // Parse capability registers
-        g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
-
-        uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
+        uint32_t hciVersion = capBase >> 16;
         KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
             << ", CapLength: " << (uint64_t)g_capLength;
 
@@ -1656,6 +1906,16 @@ namespace Drivers::USB::Xhci {
         g_rtBase = g_mmioBase + rtsOff;
         g_dbBase = g_mmioBase + dbOff;
 
+        // A firmware-described register block may be smaller than what the
+        // capability registers point into (port registers, interrupter 0,
+        // the doorbells): map those too.
+        uint64_t needed = g_capLength + OP_PORTSC_BASE + (uint64_t)((hcsParams1 >> 24) & 0xFF) * OP_PORTSC_STRIDE;
+        if (rtsOff + 0x40 > needed) needed = rtsOff + 0x40;
+        if (dbOff + 4 * ((hcsParams1 & 0xFF) + 1) > needed) needed = dbOff + 4 * ((hcsParams1 & 0xFF) + 1);
+        for (uint64_t offset = mmioSize; offset < needed; offset += 0x1000) {
+            Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
+        }
+
         KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
             << ", MaxPorts: " << (uint64_t)g_maxPorts
             << ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
@@ -1701,6 +1961,8 @@ namespace Drivers::USB::Xhci {
         // Program CONFIG
         WriteOp(OP_CONFIG, g_maxSlots);
 
+        if (!g_dmaCoherent) g_ctrlBounce = AllocateDmaBuffer(g_ctrlBouncePhys);
+
         // Allocate DCBAA
         g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
         WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
@@ -1748,13 +2010,10 @@ namespace Drivers::USB::Xhci {
         g_evtRingCCS = true;
         g_evtRingDequeue = 0;
         KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
+    }
 
-        // MSI setup
-        if (!SetupMsi(dev.Bus, dev.Device, dev.Function)) {
-            KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
-            g_pollMode = true;
-        }
-
+    // Enable the interrupter, run the controller, power and enumerate ports.
+    static bool StartController() {
         // Enable interrupter 0
         WriteRt(IR0_IMAN, IMAN_IE);
         WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
@@ -1822,11 +2081,75 @@ namespace Drivers::USB::Xhci {
             UsbDevice::EnumerateDevice(port + 1, speed);
         }
 
+        // Connected ports the scan found nothing on (still training, or the
+        // enumeration failed) get another look from the hot-plug path.
+        for (uint32_t port = 0; port < g_maxPorts; port++) {
+            if (!(ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE) & PORTSC_CCS)) continue;
+            bool active = false;
+            for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
+                if (g_devices[s].Active && g_devices[s].PortId == port + 1) active = true;
+            }
+            if (!active) {
+                g_hotplugPending[port].store(true, std::memory_order_release);
+                g_deferredWorkPending.store(true, std::memory_order_release);
+            }
+        }
+
         g_bootScanComplete = true;
         KernelLogStream(OK, "xHCI") << "Initialization complete";
         return true;
     }
 
+    bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
+                       bool activeLow, bool dmaCoherent) {
+        if (g_initialized) return false;
+
+        g_dmaCoherent = dmaCoherent;
+        InitializeController(mmioPhys, mmioSize < 0x1000 ? 0x1000 : (mmioSize + 0xFFF) & ~0xFFFULL);
+
+        uint8_t line = gsi != 0 ? Hal::GsiRouting::LineForGsi(gsi, levelTriggered, activeLow) : 0;
+        if (line != 0) {
+            Arch::RegisterIrqHandler(line, HandleInterrupt);
+            Arch::EnableLegacyIrq(line);
+            KernelLogStream(OK, "xHCI") << "Interrupt: GSI " << base::dec << (uint64_t)gsi
+                << " on platform line " << (uint64_t)line;
+        } else {
+            KernelLogStream(WARNING, "xHCI") << "No interrupt, using poll mode";
+        }
+        // Watch the event ring from the timer as well. The interrupt is a
+        // PCI INTx routed by firmware through a bridge the kernel does not
+        // drive; if it never arrives, input still works (a ring read per
+        // tick costs next to nothing).
+        g_pollMode = true;
+
+        return StartController();
+    }
+
+    bool Probe(const Pci::PciDevice& dev) {
+        if (g_initialized) return false;
+
+        KernelLogStream(OK, "xHCI") << "Found controller at PCI "
+            << base::hex << (uint64_t)dev.Bus << ":"
+            << (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
+
+        uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
+
+        KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
+
+        Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
+        KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
+
+        InitializeController(mmioPhys, 0x10000);
+
+        // MSI setup
+        if (!SetupMsi(dev.Bus, dev.Device, dev.Function) && !SetupIntx(dev.Bus, dev.Device, dev.Function)) {
+            KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
+            g_pollMode = true;
+        }
+
+        return StartController();
+    }
+
     // -------------------------------------------------------------------------
     // Initialize (standalone fallback path)
     // -------------------------------------------------------------------------
@@ -2064,8 +2387,8 @@ namespace Drivers::USB::Xhci {
         // -----------------------------------------------------------------
         // Step 12: MSI setup
         // -----------------------------------------------------------------
-        if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function)) {
-            KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
+        if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function) && !SetupIntx(foundDev->Bus, foundDev->Device, foundDev->Function)) {
+            KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
             g_pollMode = true;
         }
 

kernel/src/Drivers/USB/Xhci.hpp

@@ -177,6 +177,10 @@ namespace Drivers::USB::Xhci {
     // Device Context structures (xHCI spec section 6.2)
     // ---------------------------------------------------------------------------
 
+    // Slot Context Field0 flags
+    constexpr uint32_t SLOT_MTT        = (1u << 25);  // Multi-TT
+    constexpr uint32_t SLOT_HUB        = (1u << 26);  // the device is a hub
+
     struct SlotContext {
         uint32_t Field0;    // Route String, Speed, MTT, Hub, Context Entries
         uint32_t Field1;    // Max Exit Latency, Root Hub Port Number, Num Ports
@@ -232,8 +236,21 @@ namespace Drivers::USB::Xhci {
     struct UsbDeviceInfo {
         bool     Active;
         bool     Ready;               // descriptors/endpoints and binding are complete
-        uint8_t  PortId;
+        uint8_t  PortId;              // root hub port the device's branch hangs off
         uint32_t Speed;
+
+        // Topology (xHCI 4.3, 8.9). A device behind a hub records the hub's
+        // slot and port, its route string (one 4-bit port number per hub
+        // tier, tier 1 in bits 3:0) and, for a low/full-speed device behind a
+        // high-speed hub, the hub and port whose transaction translator
+        // carries its traffic.
+        uint8_t  ParentSlot;          // 0 = attached to a root port
+        uint8_t  ParentPort;          // port number on the parent hub
+        uint8_t  Depth;               // hubs between the device and the root port
+        uint32_t RouteString;
+        uint8_t  TtHubSlot;           // 0 = no transaction translator
+        uint8_t  TtPort;
+        bool     TtMulti;
         uint16_t VendorId;
         uint16_t ProductId;
         uint8_t  InterfaceClass;
@@ -295,6 +312,19 @@ namespace Drivers::USB::Xhci {
 
     void Initialize();
     bool Probe(const Pci::PciDevice& dev);
+
+    // A controller described by ACPI rather than found on PCI (PNP0D10,
+    // e.g. the Raspberry Pi 4's VL805 behind firmware-configured PCIe):
+    // its register block, its interrupt (an ACPI GSI, 0 = poll) and
+    // whether its DMA snoops the CPU caches (_CCA).
+    bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
+                       bool activeLow, bool dmaCoherent);
+
+    // A zeroed page for rings, contexts and buffers the controller accesses,
+    // mapped to suit the controller's DMA coherency (Memory/Dma.hpp).
+    uint8_t* AllocateDmaPage(uint64_t& outPhysAddr);
+    void* AllocateDmaPage();
+    void FreeDmaPage(void* page);
     bool IsInitialized();
     bool HasDeferredWork();
 
@@ -374,6 +404,10 @@ namespace Drivers::USB::Xhci {
     // Access device info
     UsbDeviceInfo* GetDevice(uint8_t slotId);
 
+    // Tear down a device that went away, and everything behind it if it is a
+    // hub: userspace claims, the class driver, the slot. Process context.
+    void RemoveDevice(uint8_t slotId);
+
     // Poll the event ring from process/idle context. The hard interrupt only
     // acknowledges and masks the source, then queues deferred draining.
     void PollEvents();

kernel/src/Fs/Boot.cpp

@@ -16,6 +16,8 @@ namespace Fs {
 
     namespace {
 
+        constexpr const char* RootLabel = "MONTAUKROOT";
+
         bool StringsEqual(const char* lhs, const char* rhs) {
             if (lhs == nullptr || rhs == nullptr) return false;
 
@@ -118,9 +120,15 @@ namespace Fs {
             Vfs::RegisterDrive(0, &g_ramdiskDriver);
         }
 
-        // When no ramdisk, disk partitions start at drive 0 so init.elf is found there.
+        // Without a ramdisk the root is a disk partition: the one labelled
+        // RootLabel (scripts/mk-rpi4-sd.sh makes it), else the first partition
+        // mounted, so init.elf is found at 0:/os either way.
         Fat32::RegisterProbe();
         Ext2::RegisterProbe();
+        if (!hasRamdisk && FsProbe::MountRootByLabel(RootLabel) < 0) {
+            Kt::KernelLogStream(Kt::WARNING, "Fs")
+                << "No ramdisk and no " << RootLabel << " partition; using the first one found";
+        }
         FsProbe::MountPartitions(hasRamdisk ? 1 : 0);
     }
 

kernel/src/Fs/Ext2.cpp

@@ -145,6 +145,17 @@ namespace Fs::Ext2 {
         bool     isDirectory;
     };
 
+    // An online grow in progress (see "Online grow" below): the size being
+    // grown to and how far the new block groups have been laid out.
+    struct GrowPlan {
+        bool     active;
+        uint64_t partSectors;   // partition length the plan was made for
+        uint32_t newBlocks;
+        uint32_t newGroups;
+        uint32_t newGdtBlocks;
+        uint32_t nextGroup;     // next new group to write
+    };
+
     struct Ext2Instance {
         bool     active;
         int      blockDevIndex;
@@ -186,6 +197,8 @@ namespace Fs::Ext2 {
         char (*dirNames)[MaxNameLen];
         int  dirNamesPages;
         int  dirNameCount;
+
+        GrowPlan grow;
     };
 
     // =========================================================================
@@ -1857,6 +1870,407 @@ namespace Fs::Ext2 {
         return 0;
     }
 
+    // =========================================================================
+    // Online grow
+    // =========================================================================
+    //
+    // Grows a mounted volume into its partition after the partition has been
+    // enlarged (the Raspberry Pi SD image does this on first boot, from a card
+    // image sized to its contents). The volume stays mounted: each call runs
+    // under the VFS lock and does a bounded slice of work, so other file I/O
+    // interleaves between slices.
+    //
+    // The new block groups are written first, past the old end of the volume
+    // where nothing refers to them, so an interruption there leaves the volume
+    // as it was. The commit then updates the bitmaps that change, the
+    // descriptor table and the superblock, and the superblock is the point the
+    // new size takes effect.
+    //
+    // The descriptor table grows in place, which needs free blocks right after
+    // it in every group holding a superblock copy: mke2fs reserves those for
+    // resize_inode (mk-rpi4-sd.sh asks for room up to 1 TiB). The grow uses
+    // what it needs, releases the rest and clears the feature, so a volume
+    // grows once. Volumes without the reservation can still grow while the
+    // table's last block has room (128 groups of 128 MiB at 4 KiB blocks).
+
+    static constexpr uint32_t FEATURE_COMPAT_RESIZE_INODE   = 0x0010;
+    static constexpr uint32_t FEATURE_RO_COMPAT_SPARSE_SUPER = 0x0001;
+    static constexpr uint32_t EXT2_RESIZE_INODE = 7;
+    static constexpr uint32_t SbOffsetReservedGdt = 0xCE;   // s_reserved_gdt_blocks, u16
+    // Blocks written per call before handing the VFS lock back (~4 MiB).
+    static constexpr uint32_t GrowSliceBlocks = 1024;
+    // A trailing group this close to its metadata size is left off.
+    static constexpr uint32_t MinGroupDataBlocks = 64;
+    // Write buffer for new groups, in blocks (pages at 4 KiB). A group's
+    // bitmaps and inode table are contiguous and go out in runs this long;
+    // SD cards write long runs far faster than short ones.
+    static constexpr uint32_t GrowRunBlocks = 64;
+
+    static bool GroupHasSuper(uint32_t group, bool sparse) {
+        if (!sparse || group <= 1) return true;
+        for (uint32_t base : {3u, 5u, 7u}) {
+            uint64_t power = base;
+            while (power < group) power *= base;
+            if (power == group) return true;
+        }
+        return false;
+    }
+
+    static uint32_t GdtBlocksFor(const Ext2Instance& inst, uint32_t groups) {
+        return (uint32_t)(((uint64_t)groups * sizeof(BlockGroupDescriptor)
+                           + inst.blockSize - 1) / inst.blockSize);
+    }
+
+    static uint32_t InodeTableBlocks(const Ext2Instance& inst) {
+        return (uint32_t)(((uint64_t)inst.inodesPerGroup * inst.inodeSize
+                           + inst.blockSize - 1) / inst.blockSize);
+    }
+
+    static uint32_t ReservedGdtBlocks(const uint8_t* sbRaw) {
+        auto* sb = (const Superblock*)sbRaw;
+        if (!(sb->s_feature_compat & FEATURE_COMPAT_RESIZE_INODE)) return 0;
+        uint16_t reserved;
+        memcpy(&reserved, sbRaw + SbOffsetReservedGdt, sizeof(reserved));
+        return reserved;
+    }
+
+    struct GroupLayout {
+        uint32_t start;        // first block
+        uint32_t blocks;       // blocks in the group (the last may be short)
+        uint32_t blockBitmap;
+        uint32_t inodeBitmap;
+        uint32_t inodeTable;
+        uint32_t overhead;     // metadata blocks at the start of the group
+    };
+
+    // Where a group added by the grow keeps its metadata: a superblock copy and
+    // the descriptor table where the group has one, then the two bitmaps and
+    // the inode table, as mke2fs lays out a group without flex_bg.
+    static GroupLayout NewGroupLayout(const Ext2Instance& inst, const GrowPlan& plan,
+                                      uint32_t group, bool sparse) {
+        GroupLayout l{};
+        l.start = group * inst.blocksPerGroup;
+        l.blocks = plan.newBlocks - l.start;
+        if (l.blocks > inst.blocksPerGroup) l.blocks = inst.blocksPerGroup;
+        uint32_t meta = GroupHasSuper(group, sparse) ? 1 + plan.newGdtBlocks : 0;
+        l.blockBitmap = l.start + meta;
+        l.inodeBitmap = l.blockBitmap + 1;
+        l.inodeTable = l.inodeBitmap + 1;
+        l.overhead = meta + 2 + InodeTableBlocks(inst);
+        return l;
+    }
+
+    // Work out how far the volume can grow into a partition of partSectors.
+    // Returns false when the layout is one this code does not handle; a plan
+    // whose newBlocks equals the current size means there is nothing to do.
+    static bool PlanGrow(const Ext2Instance& inst, const uint8_t* sbRaw,
+                         uint64_t partSectors, GrowPlan& plan) {
+        plan = {};
+        plan.partSectors = partSectors;
+        plan.newBlocks = inst.totalBlocks;
+        plan.newGroups = inst.groupCount;
+        plan.newGdtBlocks = GdtBlocksFor(inst, inst.groupCount);
+        plan.nextGroup = inst.groupCount;
+
+        // 1 KiB volumes start at block 1 and size groups differently; the SD
+        // image (and our own mkfs) use 4 KiB blocks.
+        if (inst.firstDataBlock != 0) return false;
+        if (inst.blocksPerGroup != inst.blockSize * 8) return false;
+        if (inst.groupCount != (inst.totalBlocks + inst.blocksPerGroup - 1) / inst.blocksPerGroup)
+            return false;
+
+        auto* sb = (const Superblock*)sbRaw;
+        bool sparse = sb->s_feature_ro_compat & FEATURE_RO_COMPAT_SPARSE_SUPER;
+        uint32_t bpg = inst.blocksPerGroup;
+        uint32_t oldGdt = GdtBlocksFor(inst, inst.groupCount);
+        uint64_t maxGroups = (uint64_t)(oldGdt + ReservedGdtBlocks(sbRaw))
+                           * (inst.blockSize / sizeof(BlockGroupDescriptor));
+        // Block and inode numbers are 32-bit.
+        if (maxGroups > 0xFFFFFFFFull / bpg) maxGroups = 0xFFFFFFFFull / bpg;
+        if (maxGroups > 0xFFFFFFFFull / inst.inodesPerGroup)
+            maxGroups = 0xFFFFFFFFull / inst.inodesPerGroup;
+
+        uint64_t blocks = partSectors / SectorsPerBlock(inst);
+        if (blocks > maxGroups * bpg) blocks = maxGroups * bpg;
+        if (blocks <= inst.totalBlocks) return true;
+
+        uint32_t groups = (uint32_t)((blocks + bpg - 1) / bpg);
+        uint32_t last = groups - 1;
+        if (last >= inst.groupCount) {
+            uint32_t lastBlocks = (uint32_t)(blocks - (uint64_t)last * bpg);
+            uint32_t gdt = GdtBlocksFor(inst, groups);
+            uint32_t overhead = (GroupHasSuper(last, sparse) ? 1 + gdt : 0)
+                              + 2 + InodeTableBlocks(inst);
+            if (lastBlocks < overhead + MinGroupDataBlocks) {
+                groups--;
+                blocks = (uint64_t)groups * bpg;
+                if (blocks <= inst.totalBlocks) return true;
+            }
+        }
+
+        plan.newBlocks = (uint32_t)blocks;
+        plan.newGroups = groups;
+        plan.newGdtBlocks = GdtBlocksFor(inst, groups);
+        return true;
+    }
+
+    static void SetBits(uint8_t* bitmap, uint32_t first, uint32_t end) {
+        for (uint32_t bit = first; bit < end; bit++) bitmap[bit / 8] |= (uint8_t)(1 << (bit % 8));
+    }
+
+    // Write blocks past the cache, then refresh any copies the cache holds.
+    // The grow writes through here because the blocks are new: there is
+    // nothing to read back, and pulling them into the LRU would only evict
+    // live ones.
+    static bool WriteGrowBlocks(Ext2Instance& inst, uint32_t first, uint32_t count,
+                                const uint8_t* data) {
+        if (!WritePartSectors(inst, BlockToPartSector(inst, first),
+                              count * SectorsPerBlock(inst), data)) return false;
+        if (inst.cache) {
+            for (uint32_t i = 0; i < count; i++) {
+                if (BlockCache::Contains(inst.cache, first + i)) {
+                    BlockCache::Write(inst.cache, first + i, data + (uint64_t)i * inst.blockSize);
+                }
+            }
+        }
+        return true;
+    }
+
+    // Write one new group's bitmaps and zeroed inode table, which NewGroupLayout
+    // places back to back, as runs of up to GrowRunBlocks from `buf`. Its
+    // superblock and descriptor copies are written at commit, once the table
+    // is final.
+    static bool PrepareGroup(Ext2Instance& inst, const GrowPlan& plan, uint32_t group,
+                             bool sparse, uint8_t* buf) {
+        GroupLayout l = NewGroupLayout(inst, plan, group, sparse);
+        uint32_t bitsPerBlock = inst.blockSize * 8;
+        uint8_t* blockBitmap = buf;
+        uint8_t* inodeBitmap = buf + inst.blockSize;
+
+        memset(buf, 0, (uint64_t)GrowRunBlocks * inst.blockSize);
+        SetBits(blockBitmap, 0, l.overhead);
+        SetBits(blockBitmap, l.blocks, bitsPerBlock);   // past the group's end
+        SetBits(inodeBitmap, inst.inodesPerGroup, bitsPerBlock);
+
+        // e2fsck reads every inode slot, so a table left with the card's old
+        // contents would show up as phantom files.
+        uint32_t total = 2 + InodeTableBlocks(inst);
+        for (uint32_t done = 0; done < total; ) {
+            uint32_t run = total - done;
+            if (run > GrowRunBlocks) run = GrowRunBlocks;
+            if (!WriteGrowBlocks(inst, l.blockBitmap + done, run, buf)) return false;
+            if (done == 0) memset(buf, 0, 2 * (uint64_t)inst.blockSize);   // zeros from here on
+            done += run;
+        }
+        return true;
+    }
+
+    // Clear `count` bits from `first` in an existing group's block bitmap and
+    // credit them to the group's free count (in memory; the caller writes the
+    // descriptor table).
+    static bool ReleaseBlocks(Ext2Instance& inst, uint32_t group, uint32_t first, uint32_t count) {
+        if (count == 0) return true;
+        uint32_t bitmapBlock = inst.bgdt[group].bg_block_bitmap;
+        if (!ReadBlock(inst, bitmapBlock, inst.blockBuf)) return false;
+        uint32_t released = 0;
+        for (uint32_t bit = first; bit < first + count; bit++) {
+            uint8_t mask = (uint8_t)(1 << (bit % 8));
+            if (inst.blockBuf[bit / 8] & mask) {
+                inst.blockBuf[bit / 8] &= (uint8_t)~mask;
+                released++;
+            }
+        }
+        if (!WriteBlock(inst, bitmapBlock, inst.blockBuf)) return false;
+        inst.bgdt[group].bg_free_blocks_count += (uint16_t)released;
+        return true;
+    }
+
+    // Superblock copy for `group` at the start of its first block, followed by
+    // the descriptor table. Group 0's superblock sits at byte 1024 instead and
+    // is written separately, last.
+    static bool WriteBackup(Ext2Instance& inst, uint32_t group, uint8_t* sbRaw,
+                            const uint8_t* gdt, uint32_t gdtBlocks) {
+        uint32_t start = group * inst.blocksPerGroup;
+        auto* sb = (Superblock*)sbRaw;
+        sb->s_block_group_nr = (uint16_t)group;
+        memset(inst.blockBuf, 0, inst.blockSize);
+        memcpy(inst.blockBuf, sbRaw, 1024);
+        sb->s_block_group_nr = 0;
+        if (!WriteGrowBlocks(inst, start, 1, inst.blockBuf)) return false;
+        return WriteGrowBlocks(inst, start + 1, gdtBlocks, gdt);
+    }
+
+    static bool CommitGrow(Ext2Instance& inst, uint8_t* sbRaw, const GrowPlan& plan) {
+        auto* sb = (Superblock*)sbRaw;
+        bool sparse = sb->s_feature_ro_compat & FEATURE_RO_COMPAT_SPARSE_SUPER;
+        uint32_t bpg = inst.blocksPerGroup;
+        uint32_t oldGroups = inst.groupCount;
+        uint32_t oldGdt = GdtBlocksFor(inst, oldGroups);
+        uint32_t reserved = ReservedGdtBlocks(sbRaw);
+
+        // ==== Bitmaps of the existing groups ====
+        // The old last group's padding bits become real blocks.
+        uint32_t lastGroup = oldGroups - 1;
+        uint32_t lastStart = lastGroup * bpg;
+        uint32_t oldLastBlocks = inst.totalBlocks - lastStart;
+        uint32_t newLastBlocks = plan.newBlocks - lastStart;
+        if (newLastBlocks > bpg) newLastBlocks = bpg;
+        if (!ReleaseBlocks(inst, lastGroup, oldLastBlocks, newLastBlocks - oldLastBlocks))
+            return false;
+
+        // The table now fills the first newGdt - oldGdt reserved blocks of every
+        // existing superblock group; the rest go back to the free pool, along
+        // with the resize inode's index block, and the feature is dropped.
+        if (reserved > 0) {
+            for (uint32_t g = 0; g < oldGroups; g++) {
+                if (!GroupHasSuper(g, sparse)) continue;
+                uint32_t firstBit = 1 + plan.newGdtBlocks;
+                uint32_t endBit = 1 + oldGdt + reserved;
+                if (!ReleaseBlocks(inst, g, firstBit, endBit - firstBit)) return false;
+            }
+
+            Inode resize;
+            if (!ReadInode(inst, EXT2_RESIZE_INODE, &resize)) return false;
+            uint32_t dind = resize.i_block[13];
+            if (dind != 0 && dind < inst.totalBlocks) {
+                if (!ReleaseBlocks(inst, dind / bpg, dind % bpg, 1)) return false;
+            }
+            memset(&resize, 0, sizeof(resize));
+            if (!WriteInode(inst, EXT2_RESIZE_INODE, &resize)) return false;
+
+            sb->s_feature_compat &= ~FEATURE_COMPAT_RESIZE_INODE;
+            uint16_t zeroReserved = 0;
+            memcpy(sbRaw + SbOffsetReservedGdt, &zeroReserved, sizeof(zeroReserved));
+        }
+
+        // ==== New descriptor table ====
+        uint32_t gdtBytes = plan.newGdtBlocks * inst.blockSize;
+        int gdtPages = (int)((gdtBytes + 0xFFF) / 0x1000);
+        auto* gdt = (BlockGroupDescriptor*)Memory::g_pfa->ReallocConsecutive(nullptr, gdtPages);
+        if (!gdt) return false;
+        memset(gdt, 0, (uint64_t)gdtPages * 0x1000);
+        memcpy(gdt, inst.bgdt, oldGroups * sizeof(BlockGroupDescriptor));
+
+        for (uint32_t g = oldGroups; g < plan.newGroups; g++) {
+            GroupLayout l = NewGroupLayout(inst, plan, g, sparse);
+            gdt[g].bg_block_bitmap = l.blockBitmap;
+            gdt[g].bg_inode_bitmap = l.inodeBitmap;
+            gdt[g].bg_inode_table = l.inodeTable;
+            gdt[g].bg_free_blocks_count = (uint16_t)(l.blocks - l.overhead);
+            gdt[g].bg_free_inodes_count = (uint16_t)inst.inodesPerGroup;
+            gdt[g].bg_used_dirs_count = 0;
+        }
+
+        uint64_t freeBlocks = 0, freeInodes = 0;
+        for (uint32_t g = 0; g < plan.newGroups; g++) {
+            freeBlocks += gdt[g].bg_free_blocks_count;
+            freeInodes += gdt[g].bg_free_inodes_count;
+        }
+
+        // ==== Superblock fields ====
+        uint64_t reservedBlocks = (uint64_t)sb->s_r_blocks_count * plan.newBlocks / inst.totalBlocks;
+        sb->s_blocks_count = plan.newBlocks;
+        sb->s_r_blocks_count = (uint32_t)reservedBlocks;
+        sb->s_inodes_count = plan.newGroups * inst.inodesPerGroup;
+        sb->s_free_blocks_count = (uint32_t)freeBlocks;
+        sb->s_free_inodes_count = (uint32_t)freeInodes;
+        sb->s_block_group_nr = 0;
+
+        // ==== Write: primary table, superblock copies, primary superblock ====
+        bool ok = WriteGrowBlocks(inst, 1, plan.newGdtBlocks, (const uint8_t*)gdt);
+        for (uint32_t g = 1; ok && g < plan.newGroups; g++) {
+            if (GroupHasSuper(g, sparse)) {
+                ok = WriteBackup(inst, g, sbRaw, (const uint8_t*)gdt, plan.newGdtBlocks);
+            }
+        }
+        ok = ok && WritePartSectors(inst, 2, 2, sbRaw);
+        if (!ok) {
+            Memory::g_pfa->Free(gdt, gdtPages);
+            return false;
+        }
+
+        auto* dev = Drivers::Storage::GetBlockDevice(inst.blockDevIndex);
+        if (dev && dev->Flush) dev->Flush(dev->Ctx);
+
+        Memory::g_pfa->Free(inst.bgdt, inst.bgdtPages);
+        inst.bgdt = gdt;
+        inst.bgdtPages = gdtPages;
+        inst.groupCount = plan.newGroups;
+        inst.totalBlocks = plan.newBlocks;
+        inst.totalInodes = plan.newGroups * inst.inodesPerGroup;
+        return true;
+    }
+
+    static void FillGrowStatus(const Ext2Instance& inst, const GrowPlan& plan,
+                               uint32_t oldGroups, bool committed, Vfs::GrowStatus* out) {
+        if (!out) return;
+        out->fsBytes = (uint64_t)inst.totalBlocks * inst.blockSize;
+        out->targetBytes = (uint64_t)plan.newBlocks * inst.blockSize;
+        // One step per new group, plus the commit.
+        out->stepsTotal = plan.newGroups - oldGroups + 1;
+        out->stepsDone = committed ? out->stepsTotal : plan.nextGroup - oldGroups;
+    }
+
+    static int GrowImpl(int instIndex, uint64_t partSectors, bool apply, Vfs::GrowStatus* out) {
+        Ext2Instance* self = InstanceAt(instIndex);
+        if (self == nullptr) return -1;
+        Ext2Instance& inst = *self;
+
+        uint8_t sbRaw[1024];
+        if (!ReadPartSectors(inst, 2, 2, sbRaw)) return -1;
+        bool sparse = ((const Superblock*)sbRaw)->s_feature_ro_compat & FEATURE_RO_COMPAT_SPARSE_SUPER;
+
+        GrowPlan plan;
+        if (inst.grow.active && inst.grow.partSectors == partSectors) {
+            plan = inst.grow;
+        } else if (!PlanGrow(inst, sbRaw, partSectors, plan)) {
+            return -2;
+        }
+
+        uint32_t oldGroups = inst.groupCount;
+        if (!apply || plan.newBlocks <= inst.totalBlocks) {
+            FillGrowStatus(inst, plan, oldGroups, false, out);
+            return 0;
+        }
+
+        if (!inst.grow.active) {
+            KernelLogStream(INFO, "Ext2") << "Growing \"" << inst.volumeLabel << "\" from "
+                << inst.totalBlocks << " to " << plan.newBlocks << " blocks ("
+                << oldGroups << " to " << plan.newGroups << " groups)";
+        }
+        plan.active = true;
+
+        int bufPages = (int)(((uint64_t)GrowRunBlocks * inst.blockSize + 0xFFF) / 0x1000);
+        uint8_t* buf = (uint8_t*)Memory::g_pfa->ReallocConsecutive(nullptr, bufPages);
+        if (!buf) return -1;
+
+        uint32_t written = 0;
+        bool ok = true;
+        while (ok && plan.nextGroup < plan.newGroups && written < GrowSliceBlocks) {
+            ok = PrepareGroup(inst, plan, plan.nextGroup, sparse, buf);
+            written += 2 + InodeTableBlocks(inst);
+            if (ok) plan.nextGroup++;
+        }
+        Memory::g_pfa->Free(buf, bufPages);
+
+        if (ok && plan.nextGroup < plan.newGroups) {
+            inst.grow = plan;
+            FillGrowStatus(inst, plan, oldGroups, false, out);
+            return 1;
+        }
+
+        inst.grow = {};
+        if (!ok || !CommitGrow(inst, sbRaw, plan)) {
+            KernelLogStream(ERROR, "Ext2") << "Grow of \"" << inst.volumeLabel << "\" failed";
+            return -1;
+        }
+
+        KernelLogStream(OK, "Ext2") << "Grew \"" << inst.volumeLabel << "\" to "
+            << inst.totalBlocks << " blocks (" << inst.groupCount << " groups)";
+        FillGrowStatus(inst, plan, oldGroups, true, out);
+        return 0;
+    }
+
     // =========================================================================
     // Template thunks — generate unique function pointers per instance
     // =========================================================================
@@ -1886,6 +2300,7 @@ namespace Fs::Ext2 {
     static const char* DrvGetFsType(void*) { return "ext2"; }
     static int DrvStat(void* c, const char* p, Vfs::StatInfo* o) { return StatImpl(CtxToInst(c), p, o); }
     static int DrvSetTimes(void* c, const char* p, int64_t a, int64_t m) { return SetTimesImpl(CtxToInst(c), p, a, m); }
+    static int DrvGrow(void* c, uint64_t s, bool a, Vfs::GrowStatus* o) { return GrowImpl(CtxToInst(c), s, a, o); }
 
     // Release everything the mount owns. The driver is either not registered
     // yet, or the VFS has deactivated its drive and drained dispatches.
@@ -2107,6 +2522,7 @@ namespace Fs::Ext2 {
             .Stat      = DrvStat,
             .SetTimes  = DrvSetTimes,
             .Unmount   = DrvUnmount,
+            .Grow      = DrvGrow,
         };
 
         KernelLogStream(OK, "Ext2") << "Mounted volume \""

kernel/src/Fs/FsProbe.cpp

@@ -78,6 +78,42 @@ namespace Fs::FsProbe {
         return 0;
     }
 
+    static bool LabelEquals(const char* label, const char* want) {
+        if (label == nullptr) return false;
+        while (*label != '\0' && *label == *want) { label++; want++; }
+        return *label == *want;
+    }
+
+    int MountRootByLabel(const char* label) {
+        if (!Vfs::IsInitialized() || g_probeCount == 0 || label == nullptr) return -1;
+        if (Vfs::IsDriveRegistered(0)) return -1;
+
+        int partCount = Drivers::Storage::Gpt::GetPartitionCount();
+        for (int i = 0; i < partCount && i < Drivers::Storage::Gpt::MaxPartitions; i++) {
+            auto* part = Drivers::Storage::Gpt::GetPartition(i);
+            if (!part || g_mounted[i] || IsEfiPartition(part)) continue;
+
+            for (int p = 0; p < g_probeCount; p++) {
+                Vfs::FsDriver* driver = g_probes[p](
+                    part->BlockDevIndex, part->StartLba, part->SectorCount);
+                if (!driver) continue;
+
+                const char* found = driver->GetLabel ? driver->GetLabel(driver->ctx) : nullptr;
+                if (!LabelEquals(found, label) || Vfs::RegisterDrive(0, driver) != 0) {
+                    DiscardUnregisteredDriver(driver);
+                    break;
+                }
+
+                g_mounted[i] = true;
+                g_driveForPart[i] = 0;
+                Kt::KernelLogStream(Kt::OK, "FsProbe") << "Mounted root partition "
+                    << i << " (" << label << ") as drive 0";
+                return i;
+            }
+        }
+        return -1;
+    }
+
     void MountPartitions(int firstDrive) {
         int partCount = Drivers::Storage::Gpt::GetPartitionCount();
         if (partCount == 0 || g_probeCount == 0) return;

kernel/src/Fs/FsProbe.hpp

@@ -18,6 +18,11 @@ namespace Fs::FsProbe {
     // Register a filesystem probe function (called once per FS driver at init).
     void Register(ProbeFn fn);
 
+    // Mount the non-EFI partition whose filesystem label is `label` as drive 0
+    // (the system root when booting without a ramdisk). Returns the global
+    // partition index, or -1 if none matched or drive 0 is already taken.
+    int MountRootByLabel(const char* label);
+
     // Probe all discovered storage partitions and auto-mount recognized filesystems.
     // Assigns each partition the lowest available VFS drive number at or above firstDrive.
     void MountPartitions(int firstDrive = 1);

kernel/src/Fs/Vfs.cpp

@@ -392,6 +392,16 @@ namespace Fs::Vfs {
         return CopyDriveString(driveNumber, &FsDriver::GetFsType, outType, maxLen);
     }
 
+    int VfsGrow(int driveNumber, uint64_t partSectors, bool apply, GrowStatus* out) {
+        vfsLock.Acquire();
+        FsDriver* driver = DriverForLocked(driveNumber);
+        int result = (driver != nullptr && driver->Grow != nullptr)
+            ? driver->Grow(driver->ctx, partSectors, apply, out)
+            : -2;
+        vfsLock.Release();
+        return result;
+    }
+
     int VfsRename(const char* oldPath, const char* newPath) {
         int oldDrive, newDrive;
         const char* oldLocal;

kernel/src/Fs/Vfs.hpp

@@ -43,6 +43,14 @@ namespace Fs::Vfs {
         bool     isDir;   // true if the entry is a directory
     };
 
+    // Progress of an online grow (FsDriver::Grow).
+    struct GrowStatus {
+        uint64_t fsBytes;       // filesystem size now
+        uint64_t targetBytes;   // size once grown; equals fsBytes if it cannot grow
+        uint32_t stepsDone;     // progress through the grow, out of stepsTotal
+        uint32_t stepsTotal;
+    };
+
     // A mounted filesystem. Every entry point takes the driver's own `ctx` so a
     // driver can serve any number of concurrent mounts from one set of function
     // pointers; drivers with a single global mount (e.g. the ramdisk) ignore it.
@@ -78,6 +86,13 @@ namespace Fs::Vfs {
         // and no dispatch can still be in flight. Drivers with statically
         // allocated state (e.g. the ramdisk) leave this null.
         void (*Unmount)(void* ctx);
+        // Optional: grow the filesystem to fill a partition of partSectors.
+        // With apply false, only report the size it would grow to. With apply
+        // true, do one bounded slice of the work: return 1 while more remains
+        // and 0 once the new size is committed or there was nothing to do.
+        // -1 is an I/O error, -2 a layout the driver cannot grow. Callers loop,
+        // so other I/O interleaves between slices.
+        int (*Grow)(void* ctx, uint64_t partSectors, bool apply, GrowStatus* out);
     };
 
     void Initialize();
@@ -108,4 +123,7 @@ namespace Fs::Vfs {
     int VfsDriveLabel(int driveNumber, char* outLabel, int maxLen);
     int VfsDriveFsType(int driveNumber, char* outType, int maxLen);
 
+    // Dispatch FsDriver::Grow for a drive; -2 if its driver cannot grow.
+    int VfsGrow(int driveNumber, uint64_t partSectors, bool apply, GrowStatus* out);
+
 }

kernel/src/Memory/Dma.cpp

@@ -0,0 +1,47 @@
+/*
+    * Dma.cpp
+    * Memory shared with bus-mastering devices
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Dma.hpp"
+#include <Arch/Cpu.hpp>
+#include <Memory/HHDM.hpp>
+#include <Memory/Paging.hpp>
+#include <Memory/PageFrameAllocator.hpp>
+
+namespace Memory::Dma {
+
+    void* AllocatePage(uint64_t& physical, bool coherent) {
+        void* page = Memory::g_pfa->AllocateZeroed();
+        if (page == nullptr) return nullptr;
+        physical = Memory::SubHHDM(page);
+
+        if (!coherent) {
+            // Push the zeroes out and drop the lines before the alias
+            // becomes uncached: a dirty line evicted later would overwrite
+            // whatever the device wrote.
+            Arch::InvalidateDataCache(page, 0x1000);
+            Memory::VMM::g_paging->MapWC(physical, (uint64_t)page);
+        }
+        return page;
+    }
+
+    void FreePage(void* page, bool coherent) {
+        if (page == nullptr) return;
+        if (!coherent) {
+            uint64_t physical = Memory::SubHHDM(page);
+            Memory::VMM::g_paging->Map(physical, (uint64_t)page);
+        }
+        Memory::g_pfa->Free(page);
+    }
+
+    void SyncForDevice(const void* buffer, uint64_t size, bool coherent) {
+        if (!coherent) Arch::CleanDataCache(buffer, size);
+    }
+
+    void SyncForCpu(const void* buffer, uint64_t size, bool coherent) {
+        if (!coherent) Arch::InvalidateDataCache(buffer, size);
+    }
+
+}

kernel/src/Memory/Dma.hpp

@@ -0,0 +1,34 @@
+/*
+    * Dma.hpp
+    * Memory shared with bus-mastering devices
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#pragma once
+#include <cstdint>
+
+// Most DMA snoops the CPU caches (all of x86, PCIe on most ARM servers,
+// QEMU). Some does not: the Raspberry Pi 4's VL805 xHCI is described with
+// ACPI _CCA = 0. A driver learns which from its bus and passes `coherent`
+// along; for coherent devices every call here is the plain allocation or a
+// no-op.
+
+namespace Memory::Dma {
+
+    // One zeroed page for descriptors and rings both sides read and write.
+    // For a non-coherent device it is mapped uncached (its HHDM alias, the
+    // kernel's only mapping of it, is switched to Normal non-cacheable), so
+    // neither side needs cache maintenance for it. Returns its HHDM address
+    // and physical address, or nullptr when out of memory.
+    void* AllocatePage(uint64_t& physical, bool coherent);
+
+    // Return a page from AllocatePage.
+    void FreePage(void* page, bool coherent);
+
+    // Streaming buffers (ordinary cacheable kernel memory a driver points a
+    // transfer at): call ForDevice before the device accesses the buffer,
+    // ForCpu after it wrote into it and before the CPU reads.
+    void SyncForDevice(const void* buffer, uint64_t size, bool coherent);
+    void SyncForCpu(const void* buffer, uint64_t size, bool coherent);
+
+}

kernel/src/Pci/Pci.cpp

@@ -690,6 +690,16 @@ namespace Pci {
     void Initialize(Hal::ACPI::CommonSDTHeader* xsdt) {
         KernelLogStream(INFO, "PCI") << "Initializing PCI subsystem";
 
+        // A plain ECAM walk of a quirky config space reads (and can fault
+        // on) host-bridge registers. Its devices are described by ACPI
+        // instead (Drivers/Init.cpp's platform devices).
+        char quirk[32];
+        if (RootBridgeEcamQuirk(quirk, sizeof(quirk))) {
+            KernelLogStream(WARNING, "PCI") << "Root bridge config space is not ECAM (quirk \""
+                << quirk << "\"): not enumerating PCI";
+            return;
+        }
+
         // Try to parse MCFG for ECAM access; fall back to legacy if unavailable
         ParseMcfg(xsdt);
 

kernel/src/Pci/Pci.hpp

@@ -91,6 +91,18 @@ namespace Pci {
     // xsdt: pointer to the XSDT (already HHDM-mapped)
     void Initialize(Hal::ACPI::CommonSDTHeader* xsdt);
 
+    // Whether the root bridge's _DSD names an ECAM quirk
+    // ("linux-ecam-quirk-id"): its config space only looks like ECAM (the
+    // Raspberry Pi 4's BCM2711 shows one device at a time through an index
+    // register). Copies the quirk id into `id`.
+    bool RootBridgeEcamQuirk(char* id, int idSize);
+
+    // Where the platform's interrupt lines are not wired by the hardware
+    // (Arch::PlatformLinesAreFixed), route every function's INTx pin from
+    // ACPI _PRT and store the assigned line in its Interrupt Line register.
+    // Needs the AML namespace; run before drivers probe.
+    void AssignInterruptLines();
+
     // =========================================================================
     // Shared PCI constants
     // =========================================================================

kernel/src/Pci/PciAcpi.cpp

@@ -0,0 +1,206 @@
+/*
+    * PciAcpi.cpp
+    * What the ACPI namespace says about PCI: INTx routing from _PRT on
+    * platforms without PC interrupt lines, and config-space quirks
+    * Copyright (c) 2026 Daniel Hammer
+*/
+
+#include "Pci.hpp"
+#include <Arch/Interrupts.hpp>
+#include <ACPI/AcpiDevices.hpp>
+#include <ACPI/GsiRouting.hpp>
+#include <ACPI/AML/AmlInterpreter.hpp>
+#include <ACPI/AML/AmlResource.hpp>
+#include <Terminal/Terminal.hpp>
+#include <CppLib/Stream.hpp>
+
+// On x86 the firmware programs every PCI function's Interrupt Line register
+// with its I/O APIC input, and drivers hand that number to EnableLegacyIrq.
+// Elsewhere nothing fills it in, so this pass does: it finds each pin's
+// global system interrupt in the root bridge's _PRT (through PCI-to-PCI
+// bridges by the standard swizzle, and through PNP0C0F link devices' _CRS),
+// binds the GSI to a platform line (GsiRouting), and writes that line into
+// the register. The drivers then work unchanged.
+
+using namespace Kt;
+using namespace Hal;
+
+namespace Pci {
+
+    static constexpr uint16_t RegInterruptPin = 0x3D;
+
+    // Extended interrupt descriptor flags
+    static constexpr uint8_t IrqFlagEdge      = 1u << 0;
+    static constexpr uint8_t IrqFlagActiveLow = 1u << 1;
+
+    struct RoutedGsi {
+        uint32_t gsi;
+        bool levelTriggered;
+        bool activeLow;
+    };
+
+    // Root-bus device and pin (0 = INTA) for a function, following
+    // PCI-to-PCI bridges up: each bridge rotates the pins by the number of
+    // the device behind it.
+    static bool ResolveToRootBus(uint8_t bus, uint8_t& device, uint8_t& pin) {
+        auto& devices = GetDevices();
+        for (int depth = 0; bus != 0 && depth < 32; depth++) {
+            const PciDevice* bridge = nullptr;
+            for (uint64_t i = 0; i < devices.size(); i++) {
+                const PciDevice& d = devices[i];
+                if (IsBridge(d) &&
+                    LegacyRead8(d.Bus, d.Device, d.Function, (uint8_t)PCI_REG_SECONDARY_BUS) == bus) {
+                    bridge = &d;
+                    break;
+                }
+            }
+            if (bridge == nullptr) return false;
+            pin = (uint8_t)((pin + device) % 4);
+            device = bridge->Device;
+            bus = bridge->Bus;
+        }
+        return bus == 0;
+    }
+
+    // The GSI a _PRT entry's source names: the source index itself when the
+    // source is 0, else the interrupt of the link device it references.
+    static bool ResolveSource(AML::Interpreter& interp, int32_t rootNode,
+                              const AML::Object& source, uint64_t sourceIndex, RoutedGsi& out) {
+        if (source.Type == AML::ObjectType::Integer && source.Integer == 0) {
+            out = RoutedGsi{ (uint32_t)sourceIndex, true, true };
+            return true;
+        }
+
+        int32_t link = interp.ResolveReference(source, rootNode);
+        if (link < 0) return false;
+
+        AML::ResourceList resources;
+        if (!AcpiDevices::EvaluateCrs(link, resources)) return false;
+
+        if (auto* ext = resources.FindFirst(AML::ResourceType::ExtendedIrq)) {
+            out = RoutedGsi{ ext->ExtendedIrq.Interrupt,
+                             (ext->ExtendedIrq.Flags & IrqFlagEdge) == 0,
+                             (ext->ExtendedIrq.Flags & IrqFlagActiveLow) != 0 };
+            return true;
+        }
+        if (auto* irq = resources.FindFirst(AML::ResourceType::Irq)) {
+            out = RoutedGsi{ irq->Irq.Irq, true, true };
+            return true;
+        }
+        return false;
+    }
+
+    static bool LookupPrt(AML::Interpreter& interp, int32_t rootNode, AML::Object& prt,
+                          uint8_t device, uint8_t pin, RoutedGsi& out) {
+        int count = interp.GetPackageCount(prt);
+        for (int i = 0; i < count; i++) {
+            AML::Object entry, address, entryPin, source, sourceIndex;
+            if (!interp.GetPackageElement(prt, i, entry) || entry.Type != AML::ObjectType::Package) continue;
+            if (!interp.GetPackageElement(entry, 0, address) ||
+                !interp.GetPackageElement(entry, 1, entryPin) ||
+                !interp.GetPackageElement(entry, 2, source) ||
+                !interp.GetPackageElement(entry, 3, sourceIndex)) continue;
+
+            // Address: device in the high word, function 0xFFFF (any).
+            if (((address.Integer >> 16) & 0xFFFF) != device || entryPin.Integer != pin) continue;
+            return ResolveSource(interp, rootNode, source, sourceIndex.Integer, out);
+        }
+        return false;
+    }
+
+    static bool FindRootBridge(AcpiDevices::DeviceInfo& root) {
+        return AcpiDevices::FindDeviceByHid("PNP0A08", root) ||
+               AcpiDevices::FindDeviceByHid("PNP0A03", root);
+    }
+
+    // Device properties (_DSD) come as { UUID, Package { Package { key,
+    // value }, ... } }; only the key/value list is looked at.
+    bool RootBridgeEcamQuirk(char* id, int idSize) {
+        auto& interp = AML::GetInterpreter();
+        if (!interp.IsInitialized()) return false;
+
+        AcpiDevices::DeviceInfo root{};
+        if (!FindRootBridge(root)) return false;
+
+        AML::Object dsd, properties;
+        if (!interp.EvaluateChild(root.NodeIndex, "_DSD", nullptr, 0, dsd) ||
+            dsd.Type != AML::ObjectType::Package ||
+            !interp.GetPackageElement(dsd, 1, properties) ||
+            properties.Type != AML::ObjectType::Package) {
+            return false;
+        }
+
+        static const char Key[] = "linux-ecam-quirk-id";
+        int count = interp.GetPackageCount(properties);
+        for (int i = 0; i < count; i++) {
+            AML::Object pair, key, value;
+            if (!interp.GetPackageElement(properties, i, pair) || pair.Type != AML::ObjectType::Package ||
+                !interp.GetPackageElement(pair, 0, key) || key.Type != AML::ObjectType::String ||
+                !interp.GetPackageElement(pair, 1, value) || value.Type != AML::ObjectType::String) {
+                continue;
+            }
+            bool match = true;
+            for (int c = 0; c < (int)sizeof(Key); c++) {
+                if (key.String.Data[c] != Key[c]) { match = false; break; }
+            }
+            if (!match) continue;
+
+            int n = 0;
+            for (; n + 1 < idSize && value.String.Data[n] != '\0'; n++) id[n] = value.String.Data[n];
+            id[n] = '\0';
+            return true;
+        }
+        return false;
+    }
+
+    void AssignInterruptLines() {
+        if (Arch::PlatformLinesAreFixed()) return;
+
+        auto& interp = AML::GetInterpreter();
+        if (!interp.IsInitialized()) return;
+
+        AcpiDevices::DeviceInfo root{};
+        if (!FindRootBridge(root)) return;
+
+        AML::Object prt;
+        if (!interp.EvaluateChild(root.NodeIndex, "_PRT", nullptr, 0, prt) ||
+            prt.Type != AML::ObjectType::Package) {
+            KernelLogStream(WARNING, "PCI") << "No _PRT on " << root.Path << ": INTx unrouted";
+            return;
+        }
+
+        int routed = 0;
+
+        auto& devices = GetDevices();
+        for (uint64_t i = 0; i < devices.size(); i++) {
+            const PciDevice& dev = devices[i];
+            uint8_t pinReg = LegacyRead8(dev.Bus, dev.Device, dev.Function, (uint8_t)RegInterruptPin);
+            if (pinReg == 0 || pinReg > 4) continue;
+
+            uint8_t device = dev.Device;
+            uint8_t pin = pinReg - 1;
+            RoutedGsi gsi{};
+            if (!ResolveToRootBus(dev.Bus, device, pin) ||
+                !LookupPrt(interp, root.NodeIndex, prt, device, pin, gsi)) {
+                KernelLogStream(WARNING, "PCI") << "No _PRT route for " << base::hex
+                    << (uint64_t)dev.Bus << ":" << (uint64_t)dev.Device << "."
+                    << (uint64_t)dev.Function << " INT" << (char)('A' + pinReg - 1);
+                continue;
+            }
+
+            uint8_t line = GsiRouting::LineForGsi(gsi.gsi, gsi.levelTriggered, gsi.activeLow);
+            if (line == 0) continue;
+
+            LegacyWrite8(dev.Bus, dev.Device, dev.Function, (uint8_t)PCI_REG_INTERRUPT, line);
+            routed++;
+            KernelLogStream(DEBUG, "PCI") << base::hex << (uint64_t)dev.Bus << ":"
+                << (uint64_t)dev.Device << "." << (uint64_t)dev.Function << " INT"
+                << (char)('A' + pinReg - 1) << " -> GSI " << base::dec << (uint64_t)gsi.gsi
+                << " (line " << (uint64_t)line << ")";
+        }
+
+        KernelLogStream(OK, "PCI") << "Routed INTx for " << base::dec << (uint64_t)routed
+            << " function(s) from _PRT";
+    }
+
+}

kernel/src/Sched/ElfLoader.cpp

@@ -179,7 +179,9 @@ namespace Sched {
     }
 
     static bool WriteUser64(uint64_t pageTableRoot, uint64_t virtualAddress, uint64_t value) {
-        uint64_t physBase = Memory::VMM::Paging::GetPhysAddr(pageTableRoot, virtualAddress);
+        // Translate the page, not the address: aarch64 returns the in-page
+        // offset too, which the line below would then add a second time.
+        uint64_t physBase = Memory::VMM::Paging::GetPhysAddr(pageTableRoot, virtualAddress & ~0xFFFULL);
         if (physBase == 0) {
             Kt::KernelLogStream(Kt::ERROR, "ELF") << "Relocation target not mapped";
             return false;