daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
93d8becce394d2da25ff08a96762aabdd6ca64e4
MontaukOS/kernel/src/Drivers/USB/Xhci.cpp
T

2607 lines
116 KiB
C++
Raw Blame History

/*
* Xhci.cpp
* xHCI (USB 3.x) Host Controller driver
* Copyright (c) 2025 Daniel Hammer
*/
#include "Xhci.hpp"
#include <Arch/Cpu.hpp>
#include <atomic>
#include "UsbDevice.hpp"
#include "HidKeyboard.hpp"
#include "HidMouse.hpp"
#include "MassStorage.hpp"
#include "Hub.hpp"
#include "UserUsb.hpp"
#include <Pci/Pci.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Dma.hpp>
#include <ACPI/GsiRouting.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Arch/Interrupts.hpp>
#include <Arch/Smp.hpp>
#include <Timekeeping/Timer.hpp>
#include <CppLib/Spinlock.hpp>
#include <atomic>
using namespace Kt;
static void BusyWaitMs(uint64_t ms) {
if (Arch::InterruptsEnabled()) {
// Interrupts enabled — use timer-based delay
uint64_t start = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - start < ms) {
Arch::Pause();
}
} else {
// Interrupts disabled (e.g. timer tick context) — the tick is not
// advancing here, so count ~1 µs delays instead
for (uint64_t i = 0; i < ms * 1000; i++) {
Arch::IoDelay();
}
}
}
namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
// Static state
// -------------------------------------------------------------------------
static bool g_initialized = false;
static bool g_bootScanComplete = false; // true after initial port scan finishes
// Hot-plug deferred work
static std::atomic<bool> g_hotplugPending[MAX_PORTS] = {};
static std::atomic<bool> g_deferredWorkPending{false};
static std::atomic<bool> g_diagnosticWorkPending{false};
static std::atomic<bool> g_hotplugProcessing{false};
static std::atomic<bool> g_irqDrainPending{false};
// HID interrupt endpoints that completed with an error. The event drain
// does not re-arm them (a halted endpoint takes commands to clear, which
// cannot be waited on from there): it records the error, and the
// deferred-work pass recovers the endpoint once g_hidRecoverDueMs is
// reached. Low/full-speed devices behind a hub's TT (every mouse on a
// Raspberry Pi 4) see transaction and split errors often enough that
// without this the device goes dead until replugged.
// g_hidErrorStreak counts errors since the last good report; it sets the
// backoff, and at HID_REENUMERATE_STREAK the device is reset and
// enumerated again, as a replug would.
static constexpr uint32_t HID_REENUMERATE_STREAK = 6;
static std::atomic<uint32_t> g_hidErrorCc[MAX_SLOTS + 1] = {}; // 0 = no recovery due
static std::atomic<uint32_t> g_hidErrorStreak[MAX_SLOTS + 1] = {};
static std::atomic<uint64_t> g_hidRecoverDueMs[MAX_SLOTS + 1] = {};
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
// a process-context mutex, so printing there can deadlock if the interrupt
// preempts a CPU that already owns the terminal lock. Capture the small
// diagnostic payload atomically and print it from ProcessDeferredWork().
static std::atomic<uint32_t> g_transferErrorsPending{0};
static std::atomic<uint32_t> g_transferErrorReports{0};
static std::atomic<uint32_t> g_lastErrorSlot{0};
static std::atomic<uint32_t> g_lastErrorEp{0};
static std::atomic<uint32_t> g_lastErrorCc{0};
static std::atomic<uint32_t> g_stormsPending{0};
static std::atomic<uint32_t> g_stormReports{0};
static std::atomic<uint32_t> g_lastStormProcessed{0};
static std::atomic<uint32_t> g_lastStormPortEvents{0};
static std::atomic<uint32_t> g_lastStormTransferEvents{0};
static std::atomic<uint32_t> g_lastStormType{0};
static std::atomic<uint32_t> g_lastStormSlot{0};
static std::atomic<uint32_t> g_lastStormEp{0};
static std::atomic<uint32_t> g_lastStormCc{0};
// MMIO region pointers
static volatile uint8_t* g_mmioBase = nullptr;
static uint8_t g_capLength = 0;
static volatile uint8_t* g_opBase = nullptr;
static volatile uint8_t* g_rtBase = nullptr;
static volatile uint8_t* g_dbBase = nullptr;
// Controller parameters
static uint32_t g_maxSlots = 0;
static uint32_t g_maxPorts = 0;
// DCBAA (Device Context Base Address Array) -- not static: accessed by UsbDevice.cpp
uint64_t* g_dcbaa = nullptr;
static uint64_t g_dcbaaPhys = 0;
// Command ring
static TRB* g_cmdRing = nullptr;
static uint64_t g_cmdRingPhys = 0;
static uint32_t g_cmdRingEnqueue = 0;
static bool g_cmdRingCCS = true;
// The controller exposes one command ring and one command-completion
// mailbox. Endpoint recovery, hot-plug enumeration, and control-transfer
// recovery can originate on different CPUs, so the whole submit/wait
// transaction must be single-owner. This is a process/idle-context lock;
// SendCommand is never called from a transfer-event callback.
static kcp::Mutex g_commandLock;
// Event ring
static TRB* g_evtRing = nullptr;
static uint64_t g_evtRingPhys = 0;
static uint32_t g_evtRingDequeue = 0;
static bool g_evtRingCCS = true;
// Event Ring Segment Table
static ERSTEntry* g_erst = nullptr;
static uint64_t g_erstPhys = 0;
// Command completion tracking
static volatile bool g_cmdCompleted = false;
static volatile uint32_t g_cmdCompletionCode = 0;
volatile uint32_t g_cmdCompletionSlotId = 0; // not static: accessed by UsbDevice.cpp
// EP0 control-transfer completion tracking -- PER SLOT. This was a single
// global, which aliased completions across devices: once userspace is up,
// another device's EP0 completion (HID idle/LED traffic, a hotplug rescan)
// released a waiting ControlTransfer EARLY, before its own status stage
// finished; the next command then rewrote TRBs over a still-executing TD
// and garbled the wire. Observed as the AX211 BT bootloader going
// permanently mute after the first FC05 whenever its bring-up ran
// post-boot (deferred) instead of during single-user early boot.
static volatile bool g_xferCompleted[MAX_SLOTS + 1] = {};
static volatile uint32_t g_xferCompletionCode[MAX_SLOTS + 1] = {};
// Synchronous bulk transfer tracking. Only one storage-style blocking bulk
// transfer is active at a time.
static volatile bool g_syncBulkActive = false;
static volatile bool g_syncBulkCompleted = false;
static volatile uint8_t g_syncBulkSlotId = 0;
static volatile uint8_t g_syncBulkEpDci = 0;
static volatile uint32_t g_syncBulkLength = 0;
static volatile uint32_t g_syncBulkResidual = 0;
static volatile uint32_t g_syncBulkCompletionCode = 0;
// Per-device info
static UsbDeviceInfo g_devices[MAX_SLOTS + 1] = {};
// True while PollEvents() is draining the event ring. Submitters that are
// reached from inside an event callback (e.g. an HCI reply sent from a
// Bluetooth event handler) see this and must NOT wait for completion --
// PollEvents is non-reentrant, so the wait would never observe the
// completion. Used by InPollContext() / ControlTransfer().
//
// MUST be a real atomic claimed with compare-exchange, not a plain
// volatile bool: PollEvents runs from syscall-context pumps (A2DP
// WriteAudio/StartSource on whatever core the app runs on) and from idle
// cores (ProcessDeferredWork). A check-then-set on a volatile is a
// cross-core TOCTOU: two cores both pass the check, both drain the shared
// event ring, g_evtRingDequeue/g_evtRingCCS desync, and completions get
// double-delivered (duplicate ACL packets) or lost
// (missed AVDTP responses) -- both observed on HW once audio streaming
// started pumping from syscall context.
static std::atomic<bool> g_pollActive{false};
// CPU index of the core currently draining the event ring in PollEvents
// (-1 = none). ControlTransfer fire-and-forgets ONLY for true same-core
// nesting (a callback invoked from THIS core's PollEvents); a different
// core merely polling must not make a process-context transfer skip its
// wait -- that returned CC_SUCCESS before the device filled the buffer
// (observed as corrupted USB control reads while another core was polling).
static std::atomic<int> g_pollOwnerCpu{-1};
// Serialises non-nested (waiting) control transfers so only one EP0
// transfer is ever in flight; the completion signal (g_xferCompleted) is a
// single global, so two concurrent waiters would otherwise consume each
// other's completion. A Mutex (interrupts stay enabled) is correct here:
// ControlTransfer is process-context only and may hold this across its
// multi-millisecond poll/wait.
static kcp::Mutex g_controlXferLock;
static int CurrentCpuIndex() {
auto* cpu = Smp::GetCurrentCpuData();
return cpu ? cpu->cpuIndex : -1;
}
// Interrupt transfer data buffers (per slot)
static uint8_t* g_interruptDataBuf[MAX_SLOTS + 1] = {};
static uint64_t g_interruptDataBufPhys[MAX_SLOTS + 1] = {};
// Bulk IN transfer data buffers (per slot)
static uint8_t* g_bulkInDataBuf[MAX_SLOTS + 1] = {};
static uint64_t g_bulkInDataBufPhys[MAX_SLOTS + 1] = {};
// Multi-buffer bulk-IN streaming pool (per slot). When PoolCount>0 the slot
// keeps that many bulk-IN transfers outstanding at all times: as each one
// completes the event handler hands its buffer to the callback and instantly
// re-arms the SAME buffer at the ring tail, so the endpoint is never without
// a place to DMA. This closes the gap that single-outstanding bulk IN leaves
// between completion and re-arm, during which the device FIFO overflows
// under sustained high-rate input. PoolCount==0 selects the legacy
// single-buffer path above (used by Bluetooth ACL), unchanged.
static constexpr uint32_t BULK_IN_POOL_MAX = 16;
static uint8_t* g_bulkInPool[MAX_SLOTS + 1][BULK_IN_POOL_MAX] = {};
static uint64_t g_bulkInPoolPhys[MAX_SLOTS + 1][BULK_IN_POOL_MAX] = {};
static uint32_t g_bulkInPoolCount[MAX_SLOTS + 1] = {}; // outstanding URBs (0=off)
static uint32_t g_bulkInPoolHead[MAX_SLOTS + 1] = {}; // next buffer to complete
static uint32_t g_bulkInPoolXferLen[MAX_SLOTS + 1] = {}; // bytes per transfer
static kcp::Spinlock g_bulkInPoolLocks[MAX_SLOTS + 1];
// Transfer callbacks for non-HID class drivers (per slot)
static TransferCallback g_transferCallbacks[MAX_SLOTS + 1] = {};
// Scratchpad buffer array
static uint64_t* g_scratchpadBufs = nullptr;
// -------------------------------------------------------------------------
// Register access helpers
// -------------------------------------------------------------------------
static void WriteOp(uint32_t reg, uint32_t value) {
*(volatile uint32_t*)(g_opBase + reg) = value;
}
// PollEvents() acknowledges the Port Status Change event, clearing PRC,
// so a controller that resets instantly (QEMU) can complete the reset
// between two reads of PRC. A port that left reset enabled is done too.
static bool PortResetComplete(uint32_t portsc) {
return (portsc & PORTSC_PRC) || ((portsc & PORTSC_PED) && !(portsc & PORTSC_PR));
}
static uint32_t ReadOp(uint32_t reg) {
return *(volatile uint32_t*)(g_opBase + reg);
}
static void WriteRt(uint32_t reg, uint32_t value) {
*(volatile uint32_t*)(g_rtBase + reg) = value;
}
static uint32_t ReadRt(uint32_t reg) {
return *(volatile uint32_t*)(g_rtBase + reg);
}
static uint32_t ReadCap(uint32_t reg) {
return *(volatile uint32_t*)(g_mmioBase + reg);
}
// TRBs (and the buffers they point at) must be visible before the
// controller is told to look.
static void WriteDoorbell(uint32_t index, uint32_t value) {
Arch::DmaWriteFence();
*(volatile uint32_t*)(g_dbBase + index * 4) = value;
}
// Hand a TRB to the controller: everything else in it first, then the
// control word that carries its cycle bit. A running ring may fetch the
// TRB the moment the cycle bit matches.
static inline void PublishTrb(TRB& trb, uint32_t control) {
Arch::DmaWriteFence();
*(volatile uint32_t*)&trb.Control = control;
}
// -------------------------------------------------------------------------
// DMA buffer allocation
// -------------------------------------------------------------------------
// Whether the controller's DMA snoops the CPU caches. PCI controllers
// are taken to be coherent; a platform controller says so in _CCA.
static bool g_dmaCoherent = true;
uint8_t* AllocateDmaPage(uint64_t& outPhysAddr) {
return (uint8_t*)Memory::Dma::AllocatePage(outPhysAddr, g_dmaCoherent);
}
void* AllocateDmaPage() {
uint64_t phys;
return AllocateDmaPage(phys);
}
void FreeDmaPage(void* page) {
Memory::Dma::FreePage(page, g_dmaCoherent);
}
static uint8_t* AllocateDmaBuffer(uint64_t& outPhysAddr) {
return AllocateDmaPage(outPhysAddr);
}
// -------------------------------------------------------------------------
// Transfer ring advance helpers (handle Link TRB wrap)
// -------------------------------------------------------------------------
// Advance EP0 ring enqueue pointer, activating Link TRB when reached
static void AdvanceEP0Ring(UsbDeviceInfo& dev) {
dev.EP0RingEnqueue++;
if (dev.EP0RingEnqueue >= XFER_RING_SIZE - 1) {
// Reached Link TRB — set its cycle bit and wrap
TRB& link = dev.EP0Ring[XFER_RING_SIZE - 1];
if (dev.EP0RingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.EP0RingCCS = !dev.EP0RingCCS;
dev.EP0RingEnqueue = 0;
}
}
// Advance interrupt ring enqueue pointer, activating Link TRB when reached
static void AdvanceInterruptRing(UsbDeviceInfo& dev) {
dev.InterruptRingEnqueue++;
if (dev.InterruptRingEnqueue >= XFER_RING_SIZE - 1) {
TRB& link = dev.InterruptRing[XFER_RING_SIZE - 1];
if (dev.InterruptRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.InterruptRingCCS = !dev.InterruptRingCCS;
dev.InterruptRingEnqueue = 0;
}
}
// Advance bulk IN ring enqueue pointer, activating Link TRB when reached
static void AdvanceBulkInRing(UsbDeviceInfo& dev) {
dev.BulkInRingEnqueue++;
if (dev.BulkInRingEnqueue >= XFER_RING_SIZE - 1) {
TRB& link = dev.BulkInRing[XFER_RING_SIZE - 1];
if (dev.BulkInRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.BulkInRingCCS = !dev.BulkInRingCCS;
dev.BulkInRingEnqueue = 0;
}
}
// Advance bulk OUT ring enqueue pointer, activating Link TRB when reached
static void AdvanceBulkOutRing(UsbDeviceInfo& dev) {
dev.BulkOutRingEnqueue++;
if (dev.BulkOutRingEnqueue >= XFER_RING_SIZE - 1) {
TRB& link = dev.BulkOutRing[XFER_RING_SIZE - 1];
if (dev.BulkOutRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
dev.BulkOutRingCCS = !dev.BulkOutRingCCS;
dev.BulkOutRingEnqueue = 0;
}
}
// -------------------------------------------------------------------------
// Forward declarations
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq, bool fromUser);
// -------------------------------------------------------------------------
// BIOS-to-OS handoff (USB Legacy Support)
//
// Per xHCI spec section 7.1: walk the extended capability list starting at
// xECP (HCCPARAMS1[31:16]) to find Cap ID 0x01 (USB Legacy Support). Set
// the OS-Owned semaphore, wait for BIOS to clear its semaphore, and then
// disable all USB-Legacy SMI sources.
//
// Without this, BIOS SMI handlers that emulate PS/2 from USB devices stay
// active and steal i8042 events, so the built-in keyboard/trackpad input
// does not reach the OS until USB activity wakes the SMI handler.
// -------------------------------------------------------------------------
static constexpr uint8_t XHCI_EXTCAP_LEGACY = 0x01;
static constexpr uint32_t USBLEGSUP_BIOS_OWNED = (1u << 16);
static constexpr uint32_t USBLEGSUP_OS_OWNED = (1u << 24);
static constexpr uint32_t USBLEGCTLSTS_SMI_ENABLES =
(1u << 0) // USB SMI Enable
| (1u << 4) // SMI on Host System Error Enable
| (1u << 13) // SMI on OS Ownership Enable
| (1u << 14) // SMI on PCI Command Enable
| (1u << 15); // SMI on BAR Enable
static constexpr uint32_t USBLEGCTLSTS_W1C_STATUS =
(1u << 20) // SMI on OS Ownership Change
| (1u << 21) // SMI on PCI Command
| (1u << 22); // SMI on BAR
static void PerformBiosHandoff() {
uint32_t hccparams1 = ReadCap(CAP_HCCPARAMS1);
uint32_t xecp = (hccparams1 >> 16) & 0xFFFF;
if (xecp == 0) {
KernelLogStream(INFO, "xHCI") << "No extended capabilities (xECP=0)";
return;
}
volatile uint32_t* extCap = (volatile uint32_t*)(g_mmioBase + xecp * 4);
for (int i = 0; i < 64; i++) {
uint32_t capReg = *extCap;
uint8_t capId = capReg & 0xFF;
uint8_t nextOffset = (capReg >> 8) & 0xFF;
if (capId == XHCI_EXTCAP_LEGACY) {
KernelLogStream(INFO, "xHCI") << "USB Legacy Support found, requesting BIOS handoff";
// Assert OS-Owned. Preserve the BIOS-Owned bit so BIOS sees the
// request and can finish cleanly before clearing its own bit.
*extCap = capReg | USBLEGSUP_OS_OWNED;
uint64_t deadline = Timekeeping::GetMilliseconds() + 1000;
bool released = false;
while (Timekeeping::GetMilliseconds() < deadline) {
if (!(*extCap & USBLEGSUP_BIOS_OWNED)) {
released = true;
break;
}
BusyWaitMs(10);
}
if (!released) {
KernelLogStream(WARNING, "xHCI")
<< "BIOS did not release controller in 1s, forcing handoff";
uint32_t v = *extCap;
v &= ~USBLEGSUP_BIOS_OWNED;
v |= USBLEGSUP_OS_OWNED;
*extCap = v;
} else {
KernelLogStream(OK, "xHCI") << "BIOS released xHCI ownership";
}
// Disable all USB-Legacy SMI sources and clear any pending
// W1C status bits. Preserves RsvdP bits via read-modify-write.
volatile uint32_t* ctlsts = extCap + 1;
uint32_t v = *ctlsts;
v &= ~USBLEGCTLSTS_SMI_ENABLES;
v |= USBLEGCTLSTS_W1C_STATUS;
*ctlsts = v;
return;
}
if (nextOffset == 0) return;
extCap += nextOffset;
}
}
// -------------------------------------------------------------------------
// MSI setup (same pattern as E1000E)
// -------------------------------------------------------------------------
// Legacy INTx, for controllers without MSI (or platforms that cannot
// receive it): the line the firmware or Pci::AssignInterruptLines put
// in the Interrupt Line register. HandleInterrupt suits a level line:
// clearing IMAN.IP drops it.
static bool SetupIntx(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t line = Pci::LegacyRead8(bus, dev, func, (uint8_t)Pci::PCI_REG_INTERRUPT);
if (line == 0 || line >= 24) return false;
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND,
pciCmd & ~Pci::PCI_CMD_INTX_DISABLE);
Arch::RegisterIrqHandler(line, HandleInterrupt);
Arch::EnableLegacyIrq(line);
KernelLogStream(OK, "xHCI") << "Using INTx: platform line " << base::dec << (uint64_t)line;
return true;
}
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability not found";
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
uint16_t msgCtrl = Pci::LegacyRead16(bus, dev, func, cap + 2);
bool is64bit = (msgCtrl & (1 << 7)) != 0;
// Message Address/Data: IRQ slot MSI_IRQ on the boot CPU
if (!Pci::WriteMsiMessage(bus, dev, func, cap, MSI_IRQ)) {
return false;
}
// Enable MSI: set bit 0 (MSI Enable), clear bits 6:4 (single message)
msgCtrl &= ~(0x70); // Clear Multiple Message Enable (bits 6:4)
msgCtrl |= (1 << 0); // MSI Enable
Pci::LegacyWrite16(bus, dev, func, cap + 2, msgCtrl);
// Disable legacy INTx in PCI command register
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
pciCmd |= Pci::PCI_CMD_INTX_DISABLE;
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND, pciCmd);
// Register the interrupt handler for MSI vector
Arch::RegisterIrqHandler(MSI_IRQ, HandleInterrupt);
KernelLogStream(OK, "xHCI") << "MSI enabled: IRQ slot " << base::dec << (uint64_t)MSI_IRQ << (is64bit ? " [64-bit]" : " [32-bit]");
return true;
}
// True when the caller is running inside PollEvents() (e.g. an HCI reply
// sent from a Bluetooth event handler). Such callers must fire-and-forget,
// not wait, since a nested PollEvents is a no-op.
bool InPollContext() {
// Same-core only: "nested under PollEvents" means an event callback on
// THIS core issued the query. A different core merely polling must
// not make callers (WaitCommandComplete & co.) skip their wait -- that
// reported success before the reply arrived (same cross-core bug
// ControlTransfer's trueNested check already guards against).
return g_pollActive.load(std::memory_order_relaxed) &&
g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
}
// -------------------------------------------------------------------------
// PollEvents - process event ring
// -------------------------------------------------------------------------
enum class PollResult : uint8_t {
Drained,
Busy,
BudgetExhausted,
};
// Without MSI nothing interrupts: the deferred-work check (timer and
// idle loop) looks at the event ring itself instead. This is the only
// mode on platforms whose PCIe has no MSI controller (Raspberry Pi 4).
static bool g_pollMode = false;
static bool EventRingHasWork() {
const volatile TRB& evt = g_evtRing[g_evtRingDequeue];
return ((evt.Control & TRB_CYCLE_BIT) != 0) == g_evtRingCCS;
}
static PollResult PollEventsBounded(uint32_t maxEvents) {
// PollEvents runs from synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and idle bottom halves on different
// CPUs. If two contexts advance g_evtRingDequeue / g_evtRingCCS, ring
// tracking desyncs and the cycle-bit check can start matching stale
// entries forever -> the boot freezes (observed wedging the Bluetooth
// firmware download at ~635 KB, where the dying device floods the event
// ring). Guard against cross-core re-entry. g_pollActive is also read by
// InPollContext() so command submitters reached from a callback can tell
// they are nested and must fire-and-forget instead of waiting.
bool expected = false;
if (!g_pollActive.compare_exchange_strong(expected, true,
std::memory_order_acquire)) {
return PollResult::Busy; // another context is draining
}
g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed);
// Bound each pass so a flooding/wedged device cannot monopolize its
// caller. Synchronous poll loops use a generous batch; the timer uses
// a much smaller recovery slice after the hard IRQ masks the source.
if (maxEvents == 0) maxEvents = 1;
uint32_t processed = 0, portEvts = 0, xferEvts = 0;
uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0;
while (processed < maxEvents) {
TRB& evt = g_evtRing[g_evtRingDequeue];
// Check if the cycle bit matches our expected cycle state
bool evtCycle = (evt.Control & TRB_CYCLE_BIT) != 0;
if (evtCycle != g_evtRingCCS) {
break; // No more events
}
// The rest of the event was written before its cycle bit.
Arch::DmaReadFence();
uint32_t trbType = (evt.Control & TRB_TYPE_MASK) >> TRB_TYPE_SHIFT;
lastType = trbType;
switch (trbType) {
case TRB_COMMAND_COMPLETION: {
uint32_t completionCode = (evt.Status >> 24) & 0xFF;
uint32_t slotId = (evt.Control >> 24) & 0xFF;
g_cmdCompletionCode = completionCode;
g_cmdCompletionSlotId = slotId;
g_cmdCompleted = true;
break;
}
case TRB_PORT_STATUS_CHANGE: {
portEvts++;
uint32_t portId = (evt.Parameter0 >> 24) & 0xFF;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE);
// Clear change bits (write-1-to-clear)
WriteOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
// Defer enumeration to ProcessDeferredWork outside interrupt
// context. Changes during the boot scan are kept too (that
// pass skips ports already enumerated): a device that
// connects after the scan looked at its port, slower on
// real hardware than in QEMU, would otherwise be lost.
if (portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
break;
}
case TRB_TRANSFER_EVENT: {
uint32_t completionCode = (evt.Status >> 24) & 0xFF;
uint32_t slotId = (evt.Control >> 24) & 0xFF;
uint32_t epDci = (evt.Control >> 16) & 0x1F;
uint32_t residual = evt.Status & 0x00FFFFFF;
xferEvts++;
lastSlot = slotId; lastEp = epDci; lastCC = completionCode;
if (epDci == 1) {
// EP0 (DCI 1) - control transfer completion, attributed
// to the slot it belongs to (see the per-slot note at
// the declaration).
if (slotId <= MAX_SLOTS) {
g_xferCompletionCode[slotId] = completionCode;
g_xferCompleted[slotId] = true;
}
} else if (slotId > 0 && slotId <= MAX_SLOTS && g_devices[slotId].Active) {
UsbDeviceInfo& dev = g_devices[slotId];
if (g_syncBulkActive &&
slotId == g_syncBulkSlotId &&
epDci == g_syncBulkEpDci) {
g_syncBulkResidual = residual;
g_syncBulkCompletionCode = completionCode;
g_syncBulkCompleted = true;
break;
}
if (completionCode == CC_SUCCESS || completionCode == CC_SHORT_PACKET) {
// Compute actual transfer length from residual
// Check if this is a bulk IN endpoint completion
uint8_t bulkInDci = dev.BulkInEpNum ? (dev.BulkInEpNum * 2 + 1) : 0;
uint8_t bulkOutDci = dev.BulkOutEpNum ? (dev.BulkOutEpNum * 2) : 0;
uint8_t intDci = dev.InterruptEpNum ? (dev.InterruptEpNum * 2 + 1) : 0;
if (epDci == bulkInDci && g_transferCallbacks[slotId]
&& g_bulkInPoolCount[slotId] > 0) {
// Multi-buffer streaming: hand back the rotating
// pool buffer and immediately re-arm it. The
// callback consumes the data synchronously (copies
// it out) before returning, so re-queuing the same
// buffer is safe -- it will not be DMA'd into again
// until the other PoolCount-1 transfers ahead of it
// complete (~PoolCount ms of slack).
g_bulkInPoolLocks[slotId].Acquire();
uint32_t poolCount = g_bulkInPoolCount[slotId];
if (poolCount > 0) {
uint32_t i = g_bulkInPoolHead[slotId];
uint32_t reqLen = g_bulkInPoolXferLen[slotId];
uint32_t len = (residual < reqLen) ? (reqLen - residual) : 0;
g_transferCallbacks[slotId](slotId, epDci,
g_bulkInPool[slotId][i], len, completionCode);
QueueBulkInTransfer(slotId, g_bulkInPool[slotId][i],
g_bulkInPoolPhys[slotId][i], reqLen);
g_bulkInPoolHead[slotId] = (i + 1) % poolCount;
}
g_bulkInPoolLocks[slotId].Release();
} else if (epDci == bulkInDci && g_transferCallbacks[slotId]) {
// Bulk IN — dispatch via registered callback.
// len = actually-transferred bytes (requested -
// residual). A 0-byte / ZLP completion has
// residual == requested, so len MUST be 0: the old
// code left len at the full max-packet and handed
// the callback a slice of STALE DMA buffer, which
// the BT driver then counted as a bogus ACL packet
// (the constant rx flood) and pushed into its RX
// ring, crowding out the real Config Response.
uint32_t reqLen = dev.BulkInMaxPacket;
uint16_t len = (residual < reqLen)
? (uint16_t)(reqLen - residual) : 0;
g_transferCallbacks[slotId](slotId, epDci,
g_bulkInDataBuf[slotId], len, completionCode);
} else if (epDci == bulkOutDci && g_transferCallbacks[slotId]) {
// Bulk OUT completion — notify callback
g_transferCallbacks[slotId](slotId, epDci,
nullptr, 0, completionCode);
} else if (epDci == intDci) {
// Interrupt IN — HID or callback dispatch.
// len = actually-transferred bytes. A ZLP has
// residual == requested, so len MUST be 0 (same
// fix as bulk IN above): the old code left len at
// the full max-packet and handed the callback a
// slice of STALE DMA buffer -- for Bluetooth that
// can replay the PREVIOUS HCI event into the
// command-complete mailbox.
uint16_t reqLen = dev.InterruptMaxPacket;
uint16_t len = (residual < reqLen)
? (uint16_t)(reqLen - residual) : 0;
if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
// Re-armed once the change is handled.
Hub::OnStatusChange((uint8_t)slotId, g_interruptDataBuf[slotId],
len, completionCode);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
HidKeyboard::ProcessReport(g_interruptDataBuf[slotId], len);
} else if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
HidMouse::ProcessReport(g_interruptDataBuf[slotId], len);
}
g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
// Re-queue for next HID report
QueueInterruptTransfer(slotId);
} else if (g_transferCallbacks[slotId]) {
// Callback is responsible for re-queuing
g_transferCallbacks[slotId](slotId, epDci,
g_interruptDataBuf[slotId], len, completionCode);
}
} else if (g_transferCallbacks[slotId]) {
// Unknown endpoint — try callback
g_transferCallbacks[slotId](slotId, epDci,
nullptr, 0, completionCode);
}
} else {
// Never print from this path: PollEvents may be
// running in the MSI handler. Defer a rate-limited
// diagnostic to idle/process context instead.
if (g_transferErrorReports.load(std::memory_order_relaxed) < 8) {
g_lastErrorSlot.store(slotId, std::memory_order_relaxed);
g_lastErrorEp.store(epDci, std::memory_order_relaxed);
g_lastErrorCc.store(completionCode, std::memory_order_relaxed);
g_transferErrorsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
if (dev.InterfaceClass == UsbDevice::CLASS_HUB && dev.InterruptEpNum &&
epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
Hub::OnStatusChange((uint8_t)slotId, nullptr, 0, completionCode);
}
if (dev.InterfaceClass == UsbDevice::CLASS_HID && dev.InterruptEpNum &&
epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
// Retry at once, then back off 10 ms doubling.
uint32_t streak = g_hidErrorStreak[slotId].fetch_add(
1, std::memory_order_relaxed) + 1;
uint64_t delay = streak <= 1 ? 0 : 10ull << (streak - 2);
if (delay > 1000) delay = 1000;
g_hidRecoverDueMs[slotId].store(
Timekeeping::GetMilliseconds() + delay, std::memory_order_relaxed);
g_hidErrorCc[slotId].store(completionCode, std::memory_order_release);
}
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
g_transferCallbacks[slotId](slotId, epDci,
nullptr, 0, completionCode);
}
}
}
break;
}
default:
break;
}
// Advance dequeue pointer
g_evtRingDequeue++;
if (g_evtRingDequeue >= EVT_RING_SIZE) {
g_evtRingDequeue = 0;
g_evtRingCCS = !g_evtRingCCS;
}
processed++;
}
// Update ERDP to tell the controller we have processed events
// Bit 3 (EHB - Event Handler Busy) must be set to clear it
uint64_t erdp = g_evtRingPhys + (uint64_t)g_evtRingDequeue * sizeof(TRB);
erdp |= (1 << 3); // Set EHB to clear it
WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32));
// A full batch means the ring is being flooded. Record the dominant
// source for a deferred, rate-limited process-context diagnostic.
bool exhausted = processed >= maxEvents;
if (exhausted) {
if (g_stormReports.load(std::memory_order_relaxed) < 8) {
g_lastStormProcessed.store(processed, std::memory_order_relaxed);
g_lastStormPortEvents.store(portEvts, std::memory_order_relaxed);
g_lastStormTransferEvents.store(xferEvts, std::memory_order_relaxed);
g_lastStormType.store(lastType, std::memory_order_relaxed);
g_lastStormSlot.store(lastSlot, std::memory_order_relaxed);
g_lastStormEp.store(lastEp, std::memory_order_relaxed);
g_lastStormCc.store(lastCC, std::memory_order_relaxed);
g_stormsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
}
g_pollOwnerCpu.store(-1, std::memory_order_relaxed);
g_pollActive.store(false, std::memory_order_release);
return exhausted ? PollResult::BudgetExhausted : PollResult::Drained;
}
void PollEvents() {
PollResult result = PollEventsBounded(EVT_RING_SIZE * 4);
if (result == PollResult::Drained &&
g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
// The hard IRQ leaves the interrupter masked. A synchronous
// process-context poll may drain the ring before an idle bottom
// half sees it, so finish the handoff here as well. Verify once
// more while masked: on another CPU, the IRQ may have acknowledged
// IMAN.IP for an event posted just after our first empty check. In
// that race, blindly enabling IE would strand the event because
// the IRQ already cleared its pending bit.
PollResult verify = PollEventsBounded(EVT_RING_SIZE * 4);
if (verify != PollResult::Drained) {
g_irqDrainPending.store(true, std::memory_order_release);
return;
}
// If hardware queues an event after this masked verification,
// IMAN.IP remains asserted and enabling IE generates a fresh MSI.
WriteRt(IR0_IMAN, IMAN_IE);
}
}
// -------------------------------------------------------------------------
// HandleInterrupt
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Clear USBSTS.EINT only (don't accidentally clear other W1C bits)
WriteOp(OP_USBSTS, USBSTS_EINT);
// Keep the hard IRQ strictly bounded. PollEvents dispatches HID,
// Bluetooth, and SDR callbacks; those callbacks can take subsystem
// locks, wake scheduler objects, re-arm endpoints, and copy sizeable
// buffers. Running them here lets a USB burst monopolize this
// higher-priority vector ahead of the LAPIC timer and can deadlock if
// an interrupted context owns one of those locks. Acknowledge and mask
// the source, then let an idle/process-context poll drain the ring.
WriteRt(IR0_IMAN, IMAN_IP);
g_irqDrainPending.store(true, std::memory_order_release);
}
// -------------------------------------------------------------------------
// SendCommand - send a command TRB on the command ring
// -------------------------------------------------------------------------
uint32_t SendCommand(const TRB& trb) {
g_commandLock.Acquire();
// Place TRB at current enqueue position
TRB& slot = g_cmdRing[g_cmdRingEnqueue];
slot.Parameter0 = trb.Parameter0;
slot.Parameter1 = trb.Parameter1;
slot.Status = trb.Status;
// Set the type and cycle bit in control
uint32_t control = trb.Control & ~TRB_CYCLE_BIT;
if (g_cmdRingCCS) {
control |= TRB_CYCLE_BIT;
}
PublishTrb(slot, control);
// Advance enqueue pointer
g_cmdRingEnqueue++;
if (g_cmdRingEnqueue >= CMD_RING_SIZE - 1) {
// We've reached the Link TRB - toggle its cycle bit and wrap
TRB& link = g_cmdRing[CMD_RING_SIZE - 1];
// Update the link TRB cycle bit to match current CCS
if (g_cmdRingCCS) {
link.Control |= TRB_CYCLE_BIT;
} else {
link.Control &= ~TRB_CYCLE_BIT;
}
g_cmdRingCCS = !g_cmdRingCCS;
g_cmdRingEnqueue = 0;
}
// Clear completion flag and ring the host controller doorbell
g_cmdCompleted = false;
WriteDoorbell(0, 0);
// Poll until command completes. Wall-clock bounded so a storm/wedge
// can't stretch this into a multi-second freeze; normal commands
// complete in well under a millisecond.
uint64_t cmdStart = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - cmdStart < 2000) {
PollEvents();
if (g_cmdCompleted) {
uint32_t result = g_cmdCompletionCode;
g_commandLock.Release();
return result;
}
// Small delay
for (int j = 0; j < 100; j++) {
Arch::CompilerBarrier();
}
}
// Do not hold the command-ring lock while taking the terminal mutex.
// A later caller may attempt recovery even if this command timed out.
g_commandLock.Release();
KernelLogStream(WARNING, "xHCI") << "Command timeout";
return 0xFF;
}
// -------------------------------------------------------------------------
// ControlTransfer - perform a control transfer on EP0
// -------------------------------------------------------------------------
// A STALL or a transaction error halts EP0 (xHCI 4.10.2.1), with the
// rest of the TD still queued; until it is reset every later request on
// the slot times out. Reset Endpoint, then Set TR Dequeue past the TD.
static void RecoverEP0(uint8_t slotId) {
UsbDeviceInfo& dev = g_devices[slotId];
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| (1 << 16); // DCI=1 (EP0)
SendCommand(resetTrb);
uint64_t newDeq = dev.EP0RingPhys + (uint64_t)dev.EP0RingEnqueue * sizeof(TRB);
if (dev.EP0RingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| (1 << 16); // DCI=1 (EP0)
SendCommand(deqTrb);
}
static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, uint64_t dataPhys, bool dirIn, bool trueNested);
// Data stages of a non-coherent controller go through this page (taken
// under g_controlXferLock). Callers pass small stack buffers, and cache
// maintenance cannot make those safe: a cache line the buffer shares with
// live stack data that the CPU dirties while the transfer runs is written
// back over what the device wrote.
static uint8_t* g_ctrlBounce = nullptr;
static uint64_t g_ctrlBouncePhys = 0;
uint32_t ControlTransfer(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, bool dirIn) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
return 0xFF;
}
// True nesting = called from a callback inside THIS core's PollEvents;
// such a call must fire-and-forget (a nested PollEvents is a no-op, so
// it could never observe its own completion). A different core merely
// polling is NOT nesting: serialise behind g_controlXferLock and wait
// normally so the global completion signal is unambiguous.
bool trueNested = g_pollActive.load(std::memory_order_relaxed) &&
g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
if (!trueNested) g_controlXferLock.Acquire();
bool hasData = data != nullptr && wLength != 0;
bool bounce = hasData && !g_dmaCoherent && !trueNested &&
g_ctrlBounce != nullptr && wLength <= 0x1000;
uint32_t cc;
if (bounce) {
// Copied in both directions: bytes a short IN transfer leaves
// alone keep the caller's contents, as with a direct transfer.
memcpy(g_ctrlBounce, data, wLength);
cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex,
wLength, g_ctrlBounce, g_ctrlBouncePhys, dirIn, false);
if (dirIn) memcpy(data, g_ctrlBounce, wLength);
} else {
// The data stage buffer is the caller's (cacheable) memory.
if (hasData) Memory::Dma::SyncForDevice(data, wLength, g_dmaCoherent);
cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex, wLength,
data, hasData ? Memory::SubHHDM(data) : 0, dirIn, trueNested);
if (hasData && dirIn) Memory::Dma::SyncForCpu(data, wLength, g_dmaCoherent);
}
if (!trueNested) g_controlXferLock.Release();
return cc;
}
static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, uint64_t dataPhys, bool dirIn, bool trueNested) {
UsbDeviceInfo& dev = g_devices[slotId];
// --- Setup Stage TRB ---
TRB& setup = dev.EP0Ring[dev.EP0RingEnqueue];
setup.Parameter0 = (uint32_t)bmRequestType | ((uint32_t)bRequest << 8) | ((uint32_t)wValue << 16);
setup.Parameter1 = (uint32_t)wIndex | ((uint32_t)wLength << 16);
setup.Status = 8; // Setup packet is always 8 bytes
uint32_t setupControl = (TRB_SETUP_STAGE << TRB_TYPE_SHIFT) | TRB_IDT;
if (wLength > 0) {
setupControl |= dirIn ? TRB_TRT_IN : TRB_TRT_OUT;
} else {
setupControl |= TRB_TRT_NODATA;
}
if (dev.EP0RingCCS) {
setupControl |= TRB_CYCLE_BIT;
}
// The Setup TRB's cycle bit is published only after the Data and
// Status TRBs are complete (below): if EP0 is still running, it
// would otherwise run into a half-built TD.
// Advance EP0 enqueue (handles Link TRB wrap)
AdvanceEP0Ring(dev);
// --- Data Stage TRB (if wLength > 0) ---
if (wLength > 0 && data != nullptr) {
TRB& dataTrb = dev.EP0Ring[dev.EP0RingEnqueue];
dataTrb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
dataTrb.Parameter1 = (uint32_t)(dataPhys >> 32);
dataTrb.Status = wLength;
uint32_t dataControl = (TRB_DATA_STAGE << TRB_TYPE_SHIFT);
if (dirIn) {
dataControl |= TRB_DIR_IN;
}
if (dev.EP0RingCCS) {
dataControl |= TRB_CYCLE_BIT;
}
PublishTrb(dataTrb, dataControl);
AdvanceEP0Ring(dev);
}
// --- Status Stage TRB ---
TRB& status = dev.EP0Ring[dev.EP0RingEnqueue];
status.Parameter0 = 0;
status.Parameter1 = 0;
status.Status = 0;
// Status stage direction is opposite of data stage
uint32_t statusControl = (TRB_STATUS_STAGE << TRB_TYPE_SHIFT) | TRB_IOC;
if (wLength > 0 && !dirIn) {
statusControl |= TRB_DIR_IN;
} else if (wLength == 0) {
statusControl |= TRB_DIR_IN;
}
if (dev.EP0RingCCS) {
statusControl |= TRB_CYCLE_BIT;
}
PublishTrb(status, statusControl);
AdvanceEP0Ring(dev);
PublishTrb(setup, setupControl);
// Ring doorbell for this slot, target EP0 (DCI 1)
g_xferCompleted[slotId] = false;
WriteDoorbell(slotId, 1);
// If we are nested inside THIS core's PollEvents (e.g. an HCI reply sent
// from a Bluetooth event handler), we cannot wait for completion here:
// the reentrancy guard makes a nested PollEvents a no-op, so the
// completion would never be observed and the timeout+recovery path would
// corrupt the EP0 ring. The transfer is submitted (doorbell rung); let
// the active PollEvents reap its completion. Fire-and-forget (no lock
// was taken for the nested case).
if (trueNested) {
return CC_SUCCESS;
}
// Poll until transfer completes. Wall-clock bounded (not iteration
// bounded) so a wedged device fails in ~2s and reports its cc, instead
// of the per-iteration cost ballooning under an event storm into a
// multi-second freeze with no output.
uint64_t xferStart = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - xferStart < 2000) {
PollEvents();
if (g_xferCompleted[slotId]) {
uint32_t cc = g_xferCompletionCode[slotId];
if (cc != CC_SUCCESS && cc != CC_SHORT_PACKET) RecoverEP0(slotId);
return cc;
}
for (int j = 0; j < 100; j++) {
Arch::CompilerBarrier();
}
}
// Callers already log their own error with request context; the xHCI
// layer just returns 0xFF and recovers EP0 so the next transfer works.
// Recover EP0 ring: Stop Endpoint, then Set TR Dequeue Pointer
// so that subsequent control transfers on this slot still work.
TRB stopTrb = {};
stopTrb.Control = (TRB_STOP_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| (1 << 16); // DCI=1 (EP0) in bits 20:16
SendCommand(stopTrb);
// Reset the enqueue pointer to the start and set the dequeue pointer
// to match so the ring is back in sync.
uint64_t newDeq = dev.EP0RingPhys
+ (uint64_t)dev.EP0RingEnqueue * sizeof(TRB);
if (dev.EP0RingCCS) {
newDeq |= 1; // DCS bit
}
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| (1 << 16); // DCI=1 (EP0)
SendCommand(deqTrb);
return 0xFF;
}
// -------------------------------------------------------------------------
// QueueInterruptTransfer
// -------------------------------------------------------------------------
void QueueInterruptTransfer(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
return;
}
UsbDeviceInfo& dev = g_devices[slotId];
// Allocate interrupt data buffer if not yet allocated
if (g_interruptDataBuf[slotId] == nullptr) {
g_interruptDataBuf[slotId] = AllocateDmaBuffer(g_interruptDataBufPhys[slotId]);
}
// Build a Normal TRB on the interrupt ring
TRB& trb = dev.InterruptRing[dev.InterruptRingEnqueue];
trb.Parameter0 = (uint32_t)(g_interruptDataBufPhys[slotId] & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(g_interruptDataBufPhys[slotId] >> 32);
trb.Status = dev.InterruptMaxPacket;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.InterruptRingCCS) {
control |= TRB_CYCLE_BIT;
}
PublishTrb(trb, control);
// Advance enqueue (handles Link TRB wrap)
AdvanceInterruptRing(dev);
// Ring doorbell: target = (InterruptEpNum * 2 + 1) for IN endpoint DCI
uint8_t target = dev.InterruptEpNum * 2 + 1;
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// ResetInterruptEndpoint - clear a halted interrupt IN endpoint and re-arm
// -------------------------------------------------------------------------
// A USB transaction error (cc=4) halts the endpoint; the host must issue
// Reset Endpoint + Set TR Dequeue before it will accept transfers again.
// Used by the Bluetooth firmware-download path, where a glitch on the event
// pipe near the end of a large upload otherwise kills event reception for
// good (no Command Complete / bootup events).
void ResetInterruptEndpoint(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
uint8_t dci = dev.InterruptEpNum * 2 + 1;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(resetTrb);
uint64_t newDeq = dev.InterruptRingPhys
+ (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
if (dev.InterruptRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
// Re-arm reception.
QueueInterruptTransfer(slotId);
}
// -------------------------------------------------------------------------
// ResetBulkInEndpoint - clear a halted bulk IN endpoint (does not re-arm)
// -------------------------------------------------------------------------
void ResetBulkInEndpoint(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (dev.BulkInEpNum == 0 || !dev.BulkInRing) return;
uint8_t dci = dev.BulkInEpNum * 2 + 1;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(resetTrb);
uint64_t newDeq = dev.BulkInRingPhys
+ (uint64_t)dev.BulkInRingEnqueue * sizeof(TRB);
if (dev.BulkInRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
}
// -------------------------------------------------------------------------
// ResetBulkOutEndpoint - clear a halted bulk OUT endpoint
// -------------------------------------------------------------------------
void ResetBulkOutEndpoint(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (dev.BulkOutEpNum == 0 || !dev.BulkOutRing) return;
uint8_t dci = dev.BulkOutEpNum * 2;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(resetTrb);
// Skip the errored TRB and everything queued behind it. The class
// driver releases the corresponding DMA slots before it sends again.
uint64_t newDeq = dev.BulkOutRingPhys
+ (uint64_t)dev.BulkOutRingEnqueue * sizeof(TRB);
if (dev.BulkOutRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
}
// -------------------------------------------------------------------------
// ResetRootPort - reset a connected root hub port again
// -------------------------------------------------------------------------
bool ResetRootPort(uint8_t portId, uint32_t& speed) {
if (!g_initialized || portId == 0 || portId > g_maxPorts) return false;
uint32_t reg = OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE;
uint32_t portsc = ReadOp(reg);
if (!(portsc & PORTSC_CCS)) return false;
WriteOp(reg, (portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
if (PortResetComplete(ReadOp(reg))) { resetDone = true; break; }
for (int j = 0; j < 100; j++) Arch::CompilerBarrier();
}
if (!resetDone) return false;
portsc = ReadOp(reg);
WriteOp(reg, (portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
if (!(portsc & PORTSC_PED)) return false;
speed = (portsc >> 10) & 0xF;
return true;
}
// -------------------------------------------------------------------------
// RecoverBulkEndpoint - clear a halt or a timed-out TRB on a bulk endpoint
// -------------------------------------------------------------------------
void RecoverBulkEndpoint(uint8_t slotId, bool dirIn) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
uint8_t epNum = dirIn ? dev.BulkInEpNum : dev.BulkOutEpNum;
TRB* ring = dirIn ? dev.BulkInRing : dev.BulkOutRing;
if (epNum == 0 || ring == nullptr) return;
uint8_t dci = dirIn ? (uint8_t)(epNum * 2 + 1) : (uint8_t)(epNum * 2);
// Reset Endpoint succeeds only on a halted endpoint; one that is still
// running (the transfer timed out) is stopped instead. Either way it
// ends up Stopped, which Set TR Dequeue requires.
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
if (!wasHalted) {
TRB stopTrb = {};
stopTrb.Control = (TRB_STOP_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(stopTrb);
}
uint64_t newDeq = dirIn
? dev.BulkInRingPhys + (uint64_t)dev.BulkInRingEnqueue * sizeof(TRB)
: dev.BulkOutRingPhys + (uint64_t)dev.BulkOutRingEnqueue * sizeof(TRB);
if (dirIn ? dev.BulkInRingCCS : dev.BulkOutRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
// Reset Endpoint put the host's data toggle back to DATA0; CLEAR_FEATURE
// (ENDPOINT_HALT) clears the device's stall and does the same on its
// side (USB 2.0 9.4.5). Without a reset the toggles still agree.
if (wasHalted) {
ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
(uint16_t)((dirIn ? 0x80 : 0x00) | epNum), 0, nullptr, false);
}
}
// -------------------------------------------------------------------------
// QueueBulkInTransfer
// -------------------------------------------------------------------------
void QueueBulkInTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.BulkInRing || dev.BulkInEpNum == 0) return;
// If caller provides nullptr, use the per-slot bulk IN DMA buffer
if (data == nullptr) {
if (g_bulkInDataBuf[slotId] == nullptr) {
g_bulkInDataBuf[slotId] = AllocateDmaBuffer(g_bulkInDataBufPhys[slotId]);
}
data = g_bulkInDataBuf[slotId];
dataPhys = g_bulkInDataBufPhys[slotId];
}
TRB& trb = dev.BulkInRing[dev.BulkInRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.BulkInRingCCS) {
control |= TRB_CYCLE_BIT;
}
PublishTrb(trb, control);
AdvanceBulkInRing(dev);
// Ring doorbell: DCI for bulk IN = EpNum * 2 + 1
uint8_t target = dev.BulkInEpNum * 2 + 1;
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// Multi-buffer bulk-IN streaming (see g_bulkInPool* declarations)
// -------------------------------------------------------------------------
// (Re)queue one transfer per pool buffer; resets the completion rotation.
// Safe to call from process context: the prime loop (a few TRB writes)
// finishes in microseconds, far below the >=tens-of-us minimum USB transfer
// time, so no completion can race the loop -- the same timing the legacy
// single-buffer start relies on.
void PrimeBulkInStream(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
g_bulkInPoolLocks[slotId].Acquire();
uint32_t n = g_bulkInPoolCount[slotId];
if (n == 0) {
g_bulkInPoolLocks[slotId].Release();
return;
}
g_bulkInPoolHead[slotId] = 0;
uint32_t len = g_bulkInPoolXferLen[slotId];
for (uint32_t i = 0; i < n; i++)
QueueBulkInTransfer(slotId, g_bulkInPool[slotId][i],
g_bulkInPoolPhys[slotId][i], len);
g_bulkInPoolLocks[slotId].Release();
}
void StartBulkInStream(uint8_t slotId, uint32_t xferLen, uint32_t numBuffers) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.BulkInRing || dev.BulkInEpNum == 0) return;
// Already armed: priming again would put a second transfer on the ring
// for every pool buffer and desync the completion rotation (callbacks
// would be handed the wrong buffer). Callers must Stop first.
if (g_bulkInPoolCount[slotId] != 0) return;
if (numBuffers < 1) numBuffers = 1;
if (numBuffers > BULK_IN_POOL_MAX) numBuffers = BULK_IN_POOL_MAX;
// One page per buffer; a single TRB must not cross a 64 KiB boundary, so
// clamp the transfer to a page (AllocateDmaBuffer hands out one page).
if (xferLen == 0 || xferLen > 4096) xferLen = 4096;
// Lazily allocate the pool; buffers persist across stop/start (same
// keep-forever pattern as g_bulkInDataBuf).
for (uint32_t i = 0; i < numBuffers; i++) {
if (g_bulkInPool[slotId][i] == nullptr) {
g_bulkInPool[slotId][i] =
AllocateDmaBuffer(g_bulkInPoolPhys[slotId][i]);
if (g_bulkInPool[slotId][i] == nullptr) {
// Out of DMA pages: fall back to however many we got.
if (i == 0) return;
numBuffers = i;
break;
}
}
}
g_bulkInPoolXferLen[slotId] = xferLen;
// Prime the transfers BEFORE publishing PoolCount so the event handler
// only switches to the rotating path once every buffer is queued.
g_bulkInPoolHead[slotId] = 0;
for (uint32_t i = 0; i < numBuffers; i++)
QueueBulkInTransfer(slotId, g_bulkInPool[slotId][i],
g_bulkInPoolPhys[slotId][i], xferLen);
g_bulkInPoolLocks[slotId].Acquire();
g_bulkInPoolCount[slotId] = numBuffers;
g_bulkInPoolLocks[slotId].Release();
}
bool IsBulkInStreamActive(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS) return false;
g_bulkInPoolLocks[slotId].Acquire();
bool active = g_bulkInPoolCount[slotId] != 0;
g_bulkInPoolLocks[slotId].Release();
return active;
}
void StopBulkInStream(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS) return;
// Disarm the rotation; any late completion now takes the (no-op for SDR)
// legacy path and is not re-armed. Buffers are retained for reuse.
g_bulkInPoolLocks[slotId].Acquire();
bool wasArmed = g_bulkInPoolCount[slotId] != 0;
g_bulkInPoolCount[slotId] = 0;
g_bulkInPoolLocks[slotId].Release();
if (!wasArmed) return;
// Flush the up-to-PoolCount TRBs still pending on the ring: Stop
// Endpoint, then Set TR Dequeue to the enqueue pointer. Without this a
// later Start would stack a fresh pool behind the stale TRBs and
// overflow the 32-entry transfer ring (16 stale + 16 new > 31 usable),
// wrapping the enqueue pointer onto still-pending TRBs.
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.Active || dev.BulkInEpNum == 0 || !dev.BulkInRing) return;
uint8_t dci = dev.BulkInEpNum * 2 + 1;
TRB stopTrb = {};
stopTrb.Control = (TRB_STOP_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(stopTrb);
uint64_t newDeq = dev.BulkInRingPhys
+ (uint64_t)dev.BulkInRingEnqueue * sizeof(TRB);
if (dev.BulkInRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
}
// -------------------------------------------------------------------------
// QueueBulkOutTransfer
// -------------------------------------------------------------------------
void QueueBulkOutTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
if (data != nullptr && length != 0) {
Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
}
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.BulkOutRing || dev.BulkOutEpNum == 0) return;
TRB& trb = dev.BulkOutRing[dev.BulkOutRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
if (dev.BulkOutRingCCS) {
control |= TRB_CYCLE_BIT;
}
PublishTrb(trb, control);
AdvanceBulkOutRing(dev);
// Ring doorbell: DCI for bulk OUT = EpNum * 2
uint8_t target = dev.BulkOutEpNum * 2;
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// BulkTransfer - blocking bulk IN/OUT transfer
// -------------------------------------------------------------------------
static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength);
uint32_t BulkTransfer(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength) {
if (data != nullptr && length != 0) {
Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
}
uint32_t cc = BulkTransferUnsynced(slotId, dirIn, data, dataPhys, length, actualLength);
if (data != nullptr && length != 0 && dirIn) {
Memory::Dma::SyncForCpu(data, length, g_dmaCoherent);
}
return cc;
}
static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength) {
if (actualLength) *actualLength = 0;
if (length == 0) return CC_SUCCESS;
if (length > 0x10000) return 0xFF;
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return 0xFF;
if (data == nullptr) return 0xFF;
if (g_syncBulkActive) return 0xFF;
UsbDeviceInfo& dev = g_devices[slotId];
if (dataPhys == 0) dataPhys = Memory::SubHHDM(data);
uint8_t target = 0;
if (dirIn) {
if (!dev.BulkInRing || dev.BulkInEpNum == 0) return 0xFF;
TRB& trb = dev.BulkInRing[dev.BulkInRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.BulkInRingCCS) control |= TRB_CYCLE_BIT;
PublishTrb(trb, control);
AdvanceBulkInRing(dev);
target = dev.BulkInEpNum * 2 + 1;
} else {
if (!dev.BulkOutRing || dev.BulkOutEpNum == 0) return 0xFF;
TRB& trb = dev.BulkOutRing[dev.BulkOutRingEnqueue];
trb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
trb.Parameter1 = (uint32_t)(dataPhys >> 32);
trb.Status = length;
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
if (dev.BulkOutRingCCS) control |= TRB_CYCLE_BIT;
PublishTrb(trb, control);
AdvanceBulkOutRing(dev);
target = dev.BulkOutEpNum * 2;
}
g_syncBulkSlotId = slotId;
g_syncBulkEpDci = target;
g_syncBulkLength = length;
g_syncBulkResidual = length;
g_syncBulkCompletionCode = 0;
g_syncBulkCompleted = false;
g_syncBulkActive = true;
WriteDoorbell(slotId, target);
uint64_t start = Timekeeping::GetMilliseconds();
while (Timekeeping::GetMilliseconds() - start < 5000) {
PollEvents();
if (g_syncBulkCompleted) {
uint32_t cc = g_syncBulkCompletionCode;
uint32_t residual = g_syncBulkResidual;
g_syncBulkActive = false;
if (actualLength) {
*actualLength = (residual < g_syncBulkLength)
? (g_syncBulkLength - residual)
: 0;
}
return cc;
}
for (int j = 0; j < 100; j++) {
Arch::CompilerBarrier();
}
}
g_syncBulkActive = false;
KernelLogStream(WARNING, "xHCI") << "Bulk transfer timeout on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)target;
return 0xFF;
}
// -------------------------------------------------------------------------
// RegisterTransferCallback
// -------------------------------------------------------------------------
void RegisterTransferCallback(uint8_t slotId, TransferCallback cb) {
if (slotId > 0 && slotId <= MAX_SLOTS) {
g_transferCallbacks[slotId] = cb;
}
}
// -------------------------------------------------------------------------
// RingDoorbell
// -------------------------------------------------------------------------
void RingDoorbell(uint8_t slotId, uint8_t target) {
WriteDoorbell(slotId, target);
}
// -------------------------------------------------------------------------
// GetDevice
// -------------------------------------------------------------------------
UsbDeviceInfo* GetDevice(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS) {
return nullptr;
}
return &g_devices[slotId];
}
// -------------------------------------------------------------------------
// IsInitialized
// -------------------------------------------------------------------------
bool IsInitialized() {
return g_initialized;
}
// A HID endpoint whose error backoff has run out.
static bool HidRecoveryDue() {
uint64_t now = 0;
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
if (now == 0) now = Timekeeping::GetMilliseconds();
if (now >= g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) return true;
}
return false;
}
bool HasDeferredWork() {
if (g_initialized && g_pollMode && EventRingHasWork()) {
g_irqDrainPending.store(true, std::memory_order_release);
}
return g_initialized &&
(g_irqDrainPending.load(std::memory_order_acquire) ||
g_deferredWorkPending.load(std::memory_order_acquire) ||
(g_bootScanComplete && (Hub::HasPendingWork() || HidRecoveryDue())) ||
g_diagnosticWorkPending.load(std::memory_order_acquire));
}
void ServiceDeferredIrqEvents(uint32_t budget) {
if (!g_initialized ||
!g_irqDrainPending.load(std::memory_order_acquire)) return;
PollResult result = PollEventsBounded(budget);
if (result != PollResult::Drained) return;
// Clear before re-enabling. If hardware queued an event after the ring
// appeared empty, IMAN.IP remains asserted and enabling IE generates a
// fresh MSI; no event can be lost in this handoff.
if (g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
WriteRt(IR0_IMAN, IMAN_IE);
}
}
static void PrintDeferredDiagnostics() {
// Clear the wakeup before consuming counters. A producer racing after
// this store sets it again, so deferred work cannot be lost.
g_diagnosticWorkPending.store(false, std::memory_order_release);
uint32_t errors = g_transferErrorsPending.exchange(0, std::memory_order_acq_rel);
if (errors != 0 &&
g_transferErrorReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)errors
<< " transfer error(s); last slot="
<< (uint64_t)g_lastErrorSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastErrorEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastErrorCc.load(std::memory_order_relaxed);
}
uint32_t storms = g_stormsPending.exchange(0, std::memory_order_acq_rel);
if (storms != 0 && g_stormReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)storms
<< " event storm(s); last batch="
<< (uint64_t)g_lastStormProcessed.load(std::memory_order_relaxed)
<< " (port=" << (uint64_t)g_lastStormPortEvents.load(std::memory_order_relaxed)
<< " xfer=" << (uint64_t)g_lastStormTransferEvents.load(std::memory_order_relaxed)
<< " type=" << (uint64_t)g_lastStormType.load(std::memory_order_relaxed)
<< " slot=" << (uint64_t)g_lastStormSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastStormEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastStormCc.load(std::memory_order_relaxed) << ")";
}
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
Hub::UnregisterHub(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
MassStorage::UnregisterDevice(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
HidKeyboard::UnregisterDevice(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
HidMouse::UnregisterDevice(slotId);
}
}
static void DisableSlot(uint8_t slotId) {
TRB trb = {};
trb.Control = (TRB_DISABLE_SLOT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24);
uint32_t cc = SendCommand(trb);
if (cc != CC_SUCCESS) {
KernelLogStream(WARNING, "xHCI") << "Disable Slot failed for slot "
<< base::dec << (uint64_t)slotId << " cc=" << (uint64_t)cc;
}
}
void RemoveDevice(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
// Children first: a hub's slot outlives the devices addressed through it.
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (s != slotId && g_devices[s].Active && g_devices[s].ParentSlot == slotId) {
RemoveDevice(s);
}
}
UserUsb::DeviceDisconnected(slotId);
UnregisterClassDriver(slotId, g_devices[slotId]);
g_devices[slotId].Active = false;
g_hidErrorCc[slotId].store(0, std::memory_order_relaxed);
g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
g_transferCallbacks[slotId] = nullptr;
DisableSlot(slotId);
g_dcbaa[slotId] = 0;
g_devices[slotId] = {};
KernelLogStream(INFO, "xHCI") << "Slot " << base::dec << (uint64_t)slotId << " deactivated";
}
// Bring a HID device's interrupt endpoint back after a failed transfer.
// Reset Endpoint clears a halt (a Context State Error when the error did
// not halt it), Set TR Dequeue skips the failed TRB, and CLEAR_FEATURE
// (ENDPOINT_HALT) clears a stall on the device and puts its data toggle
// back to DATA0 to match the reset host side (USB 2.0 9.4.5). If errors keep coming, the
// device is removed and its port reset, as a replug would.
static void RecoverHidEndpoint(uint8_t slotId, uint32_t cc) {
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.Active || dev.InterfaceClass != UsbDevice::CLASS_HID ||
dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
uint32_t streak = g_hidErrorStreak[slotId].load(std::memory_order_relaxed);
if (streak >= HID_REENUMERATE_STREAK) {
KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
<< ": HID endpoint keeps failing (cc=" << (uint64_t)cc
<< "), resetting the device";
uint8_t parentSlot = dev.ParentSlot, parentPort = dev.ParentPort;
uint8_t rootPort = dev.PortId;
RemoveDevice(slotId);
if (parentSlot != 0) {
Hub::RequestPortCheck(parentSlot, parentPort);
} else if (rootPort >= 1 && rootPort <= g_maxPorts) {
// Disable the port (PED is write-1-to-clear) so the hot-plug
// pass resets it before enumerating.
uint32_t reg = OP_PORTSC_BASE + (rootPort - 1) * OP_PORTSC_STRIDE;
WriteOp(reg, (ReadOp(reg) & PORTSC_PRESERVE) | PORTSC_PED);
g_hotplugPending[rootPort - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
return;
}
if (streak <= 1) {
KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
<< ": HID transfer failed (cc=" << (uint64_t)cc << "), recovering endpoint";
}
uint8_t dci = dev.InterruptEpNum * 2 + 1;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
uint64_t newDeq = dev.InterruptRingPhys
+ (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
if (dev.InterruptRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
// Only after a real reset: the host toggle is untouched otherwise.
if (wasHalted) {
ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
(uint16_t)(0x80 | dev.InterruptEpNum), 0, nullptr, false);
}
if (dev.Active) QueueInterruptTransfer(slotId);
}
static void ProcessHidRecovery() {
uint64_t now = Timekeeping::GetMilliseconds();
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
if (now < g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) continue;
uint32_t cc = g_hidErrorCc[s].exchange(0, std::memory_order_acq_rel);
if (cc != 0) RecoverHidEndpoint(s, cc);
}
}
// -------------------------------------------------------------------------
// ProcessDeferredWork - handle hot-plug outside interrupt context.
// Called from the BSP idle path so class-driver probing can wait on timers.
// -------------------------------------------------------------------------
void ProcessDeferredWork() {
if (!g_initialized) return;
// The hard IRQ only acknowledges and masks the interrupter. Drain a
// bounded slice here, outside interrupt context, so callbacks may use
// their normal locks without blocking the timer or scheduler. A busy
// process that is synchronously waiting for USB calls PollEvents()
// itself, which performs the same unmask handoff when the ring is dry.
ServiceDeferredIrqEvents(64);
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
bool haveHotplug = g_bootScanComplete &&
(g_deferredWorkPending.load(std::memory_order_acquire) || Hub::HasPendingWork() ||
HidRecoveryDue());
if (!haveDiagnostics && !haveHotplug) return;
// CAS claim: any idle core may call this, but only one runs at a time.
// AcquireTransport in MassStorage handles the SCSI-in-flight case itself.
bool expected = false;
if (!g_hotplugProcessing.compare_exchange_strong(expected, true,
std::memory_order_acquire, std::memory_order_relaxed)) {
return;
}
if (haveDiagnostics) PrintDeferredDiagnostics();
if (!haveHotplug) {
g_hotplugProcessing.store(false, std::memory_order_release);
return;
}
// First: a device it gives up on is queued for the hot-plug pass below.
ProcessHidRecovery();
g_deferredWorkPending.store(false, std::memory_order_release);
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!g_hotplugPending[port].exchange(false, std::memory_order_acq_rel)) continue;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (portsc & PORTSC_CCS) {
// Device connected — check if already assigned to a slot
bool alreadyActive = false;
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
alreadyActive = true;
break;
}
}
if (alreadyActive) continue;
if (portsc & PORTSC_PED) {
// Already enabled — enumerate after recovery delay
uint32_t speed = (portsc >> 10) & 0xF;
BusyWaitMs(10);
UsbDevice::EnumerateDevice(port + 1, speed);
} else {
// Need port reset first
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (PortResetComplete(ps)) {
resetDone = true;
break;
}
for (int j = 0; j < 100; j++) {
Arch::CompilerBarrier();
}
}
if (!resetDone) {
KernelLogStream(WARNING, "xHCI") << "Hot-plug: port "
<< base::dec << (uint64_t)(port + 1) << " reset timeout";
continue;
}
portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
uint32_t speed = (portsc >> 10) & 0xF;
// Clear change bits
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
// Post-reset recovery delay (USB spec requires >= 10ms)
BusyWaitMs(10);
UsbDevice::EnumerateDevice(port + 1, speed);
}
} else {
// Device disconnected: deactivate its slot (and, for a hub,
// the slots of everything behind it)
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1 &&
g_devices[s].ParentSlot == 0) {
KernelLogStream(INFO, "xHCI") << "Hot-unplug: port "
<< base::dec << (uint64_t)(port + 1);
RemoveDevice(s);
break;
}
}
}
}
// Port changes behind hubs.
Hub::ProcessPendingWork();
g_hotplugProcessing.store(false, std::memory_order_release);
}
// -------------------------------------------------------------------------
// Probe (called by PCI driver matching framework)
// -------------------------------------------------------------------------
// Map the register block, reset the controller and set up its
// context and command/event rings. Interrupt delivery is the caller's.
static void InitializeController(uint64_t mmioPhys, uint64_t mmioSize) {
for (uint64_t offset = 0; offset < mmioSize; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
// Parse capability registers. CAPLENGTH and HCIVERSION share the
// first dword; read it whole, since narrower accesses are not safe
// behind every PCIe bridge (Linux does the same).
uint32_t capBase = ReadCap(CAP_CAPLENGTH);
g_capLength = (uint8_t)(capBase & 0xFF);
uint32_t hciVersion = capBase >> 16;
KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
<< ", CapLength: " << (uint64_t)g_capLength;
uint32_t hcsParams1 = ReadCap(CAP_HCSPARAMS1);
g_maxSlots = hcsParams1 & 0xFF;
g_maxPorts = (hcsParams1 >> 24) & 0xFF;
uint32_t hcsParams2 = ReadCap(CAP_HCSPARAMS2);
uint32_t scratchpadBufsHi = (hcsParams2 >> 21) & 0x1F;
uint32_t scratchpadBufsLo = (hcsParams2 >> 27) & 0x1F;
uint32_t maxScratchpadBufs = (scratchpadBufsHi << 5) | scratchpadBufsLo;
uint32_t dbOff = ReadCap(CAP_DBOFF) & ~0x3u;
uint32_t rtsOff = ReadCap(CAP_RTSOFF) & ~0x1Fu;
g_opBase = g_mmioBase + g_capLength;
g_rtBase = g_mmioBase + rtsOff;
g_dbBase = g_mmioBase + dbOff;
// A firmware-described register block may be smaller than what the
// capability registers point into (port registers, interrupter 0,
// the doorbells): map those too.
uint64_t needed = g_capLength + OP_PORTSC_BASE + (uint64_t)((hcsParams1 >> 24) & 0xFF) * OP_PORTSC_STRIDE;
if (rtsOff + 0x40 > needed) needed = rtsOff + 0x40;
if (dbOff + 4 * ((hcsParams1 & 0xFF) + 1) > needed) needed = dbOff + 4 * ((hcsParams1 & 0xFF) + 1);
for (uint64_t offset = mmioSize; offset < needed; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
<< ", MaxPorts: " << (uint64_t)g_maxPorts
<< ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
if (g_maxSlots > MAX_SLOTS) g_maxSlots = MAX_SLOTS;
if (g_maxPorts > MAX_PORTS) g_maxPorts = MAX_PORTS;
// Tell BIOS to release the controller and disable USB-Legacy SMIs.
// Must run before we halt/reset so BIOS gets a chance to shut down
// its own use of the controller cleanly.
PerformBiosHandoff();
// Halt controller
uint32_t usbcmd = ReadOp(OP_USBCMD);
usbcmd &= ~USBCMD_RS;
WriteOp(OP_USBCMD, usbcmd);
for (uint32_t i = 0; i < 100000; i++) {
if (ReadOp(OP_USBSTS) & USBSTS_HCH) break;
for (int j = 0; j < 10; j++) Arch::CompilerBarrier();
}
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) {
KernelLogStream(WARNING, "xHCI") << "Controller failed to halt";
}
KernelLogStream(OK, "xHCI") << "Controller halted";
// Reset controller
WriteOp(OP_USBCMD, USBCMD_HCRST);
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBCMD) & USBCMD_HCRST)) break;
for (int j = 0; j < 10; j++) Arch::CompilerBarrier();
}
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_CNR)) break;
for (int j = 0; j < 10; j++) Arch::CompilerBarrier();
}
if (ReadOp(OP_USBSTS) & USBSTS_CNR) {
KernelLogStream(WARNING, "xHCI") << "Controller not ready after reset";
}
KernelLogStream(OK, "xHCI") << "Controller reset complete";
// Program CONFIG
WriteOp(OP_CONFIG, g_maxSlots);
if (!g_dmaCoherent) g_ctrlBounce = AllocateDmaBuffer(g_ctrlBouncePhys);
// Allocate DCBAA
g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
WriteOp(OP_DCBAAP + 4, (uint32_t)(g_dcbaaPhys >> 32));
KernelLogStream(OK, "xHCI") << "DCBAA at phys " << base::hex << g_dcbaaPhys;
// Scratchpad buffers
if (maxScratchpadBufs > 0) {
uint64_t spArrayPhys;
g_scratchpadBufs = (uint64_t*)AllocateDmaBuffer(spArrayPhys);
for (uint32_t i = 0; i < maxScratchpadBufs; i++) {
uint64_t bufPhys;
AllocateDmaBuffer(bufPhys);
g_scratchpadBufs[i] = bufPhys;
}
g_dcbaa[0] = spArrayPhys;
KernelLogStream(OK, "xHCI") << "Allocated " << base::dec << (uint64_t)maxScratchpadBufs << " scratchpad buffers";
}
// Command ring
g_cmdRing = (TRB*)AllocateDmaBuffer(g_cmdRingPhys);
TRB& linkTrb = g_cmdRing[CMD_RING_SIZE - 1];
linkTrb.Parameter0 = (uint32_t)(g_cmdRingPhys & 0xFFFFFFFF);
linkTrb.Parameter1 = (uint32_t)(g_cmdRingPhys >> 32);
linkTrb.Status = 0;
linkTrb.Control = (TRB_LINK << TRB_TYPE_SHIFT) | TRB_ENT;
uint64_t crcr = g_cmdRingPhys | TRB_CYCLE_BIT;
WriteOp(OP_CRCR, (uint32_t)(crcr & 0xFFFFFFFF));
WriteOp(OP_CRCR + 4, (uint32_t)(crcr >> 32));
g_cmdRingCCS = true;
g_cmdRingEnqueue = 0;
KernelLogStream(OK, "xHCI") << "Command ring at phys " << base::hex << g_cmdRingPhys;
// Event ring + ERST
g_evtRing = (TRB*)AllocateDmaBuffer(g_evtRingPhys);
g_erst = (ERSTEntry*)AllocateDmaBuffer(g_erstPhys);
g_erst[0].RingSegmentBase = g_evtRingPhys;
g_erst[0].RingSegmentSize = EVT_RING_SIZE;
g_erst[0].Reserved = 0;
WriteRt(IR0_ERSTSZ, 1);
WriteRt(IR0_ERDP, (uint32_t)(g_evtRingPhys & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(g_evtRingPhys >> 32));
WriteRt(IR0_ERSTBA, (uint32_t)(g_erstPhys & 0xFFFFFFFF));
WriteRt(IR0_ERSTBA + 4, (uint32_t)(g_erstPhys >> 32));
g_evtRingCCS = true;
g_evtRingDequeue = 0;
KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
}
// Enable the interrupter, run the controller, power and enumerate ports.
static bool StartController() {
// Enable interrupter 0
WriteRt(IR0_IMAN, IMAN_IE);
WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
// Start controller
WriteOp(OP_USBCMD, USBCMD_RS | USBCMD_INTE | USBCMD_HSEE);
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) break;
for (int j = 0; j < 10; j++) Arch::CompilerBarrier();
}
KernelLogStream(OK, "xHCI") << "Controller started";
g_initialized = true;
// Power on all ports
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (!(portsc & PORTSC_PP)) {
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE, PORTSC_PP);
}
}
BusyWaitMs(20);
KernelLogStream(OK, "xHCI") << "All ports powered";
// Port scanning
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (!(portsc & PORTSC_CCS)) continue;
KernelLogStream(INFO, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": device connected, PORTSC=" << base::hex << (uint64_t)portsc;
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (PortResetComplete(ps)) { resetDone = true; break; }
for (int j = 0; j < 100; j++) Arch::CompilerBarrier();
}
if (!resetDone) {
KernelLogStream(WARNING, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1) << " reset timeout";
continue;
}
portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
uint32_t speed = (portsc >> 10) & 0xF;
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
const char* speedStr = "Unknown";
switch (speed) {
case SPEED_FULL: speedStr = "Full (12 Mbps)"; break;
case SPEED_LOW: speedStr = "Low (1.5 Mbps)"; break;
case SPEED_HIGH: speedStr = "High (480 Mbps)"; break;
case SPEED_SUPER: speedStr = "Super (5 Gbps)"; break;
}
KernelLogStream(OK, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": reset complete, speed=" << speedStr;
BusyWaitMs(10);
UsbDevice::EnumerateDevice(port + 1, speed);
}
// Connected ports the scan found nothing on (still training, or the
// enumeration failed) get another look from the hot-plug path.
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!(ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE) & PORTSC_CCS)) continue;
bool active = false;
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1) active = true;
}
if (!active) {
g_hotplugPending[port].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
}
g_bootScanComplete = true;
KernelLogStream(OK, "xHCI") << "Initialization complete";
return true;
}
bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
bool activeLow, bool dmaCoherent) {
if (g_initialized) return false;
g_dmaCoherent = dmaCoherent;
InitializeController(mmioPhys, mmioSize < 0x1000 ? 0x1000 : (mmioSize + 0xFFF) & ~0xFFFULL);
uint8_t line = gsi != 0 ? Hal::GsiRouting::LineForGsi(gsi, levelTriggered, activeLow) : 0;
if (line != 0) {
Arch::RegisterIrqHandler(line, HandleInterrupt);
Arch::EnableLegacyIrq(line);
KernelLogStream(OK, "xHCI") << "Interrupt: GSI " << base::dec << (uint64_t)gsi
<< " on platform line " << (uint64_t)line;
} else {
KernelLogStream(WARNING, "xHCI") << "No interrupt, using poll mode";
}
// Watch the event ring from the timer as well. The interrupt is a
// PCI INTx routed by firmware through a bridge the kernel does not
// drive; if it never arrives, input still works (a ring read per
// tick costs next to nothing).
g_pollMode = true;
return StartController();
}
bool Probe(const Pci::PciDevice& dev) {
if (g_initialized) return false;
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
<< base::hex << (uint64_t)dev.Bus << ":"
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
InitializeController(mmioPhys, 0x10000);
// MSI setup
if (!SetupMsi(dev.Bus, dev.Device, dev.Function) && !SetupIntx(dev.Bus, dev.Device, dev.Function)) {
KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
g_pollMode = true;
}
return StartController();
}
// -------------------------------------------------------------------------
// Initialize (standalone fallback path)
// -------------------------------------------------------------------------
void Initialize() {
KernelLogStream(INFO, "xHCI") << "Scanning for xHCI controller...";
// -----------------------------------------------------------------
// Step 1: Find xHCI controller on PCI bus
// -----------------------------------------------------------------
auto& devices = Pci::GetDevices();
const Pci::PciDevice* foundDev = nullptr;
for (uint64_t i = 0; i < devices.size(); i++) {
if (devices[i].ClassCode == PCI_CLASS_SERIAL &&
devices[i].SubClass == PCI_SUBCLASS_USB &&
devices[i].ProgIf == PCI_PROGIF_XHCI) {
foundDev = &devices[i];
break;
}
}
if (foundDev == nullptr) {
KernelLogStream(WARNING, "xHCI") << "No xHCI controller found";
return;
}
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
<< base::hex << (uint64_t)foundDev->Bus << ":"
<< (uint64_t)foundDev->Device << "." << (uint64_t)foundDev->Function;
// -----------------------------------------------------------------
// Step 2: Read BAR0 and map MMIO region
// -----------------------------------------------------------------
uint64_t mmioPhys = Pci::ReadBar0(foundDev->Bus, foundDev->Device, foundDev->Function);
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
// Map 64KB (16 pages) of MMIO space
constexpr uint64_t MmioSize = 0x10000;
for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
// -----------------------------------------------------------------
// Step 3: Enable PCI bus master and memory space
// -----------------------------------------------------------------
Pci::EnableBusMaster(foundDev->Bus, foundDev->Device, foundDev->Function);
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
// -----------------------------------------------------------------
// Step 4: Parse capability registers
// -----------------------------------------------------------------
g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
<< ", CapLength: " << (uint64_t)g_capLength;
uint32_t hcsParams1 = ReadCap(CAP_HCSPARAMS1);
g_maxSlots = hcsParams1 & 0xFF;
g_maxPorts = (hcsParams1 >> 24) & 0xFF;
uint32_t hcsParams2 = ReadCap(CAP_HCSPARAMS2);
uint32_t scratchpadBufsHi = (hcsParams2 >> 21) & 0x1F;
uint32_t scratchpadBufsLo = (hcsParams2 >> 27) & 0x1F;
uint32_t maxScratchpadBufs = (scratchpadBufsHi << 5) | scratchpadBufsLo;
uint32_t dbOff = ReadCap(CAP_DBOFF) & ~0x3u;
uint32_t rtsOff = ReadCap(CAP_RTSOFF) & ~0x1Fu;
g_opBase = g_mmioBase + g_capLength;
g_rtBase = g_mmioBase + rtsOff;
g_dbBase = g_mmioBase + dbOff;
KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
<< ", MaxPorts: " << (uint64_t)g_maxPorts
<< ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
// Cap slots to our maximum
if (g_maxSlots > MAX_SLOTS) {
g_maxSlots = MAX_SLOTS;
}
if (g_maxPorts > MAX_PORTS) {
g_maxPorts = MAX_PORTS;
}
// -----------------------------------------------------------------
// Step 4.5: BIOS-to-OS handoff (USB Legacy Support)
// -----------------------------------------------------------------
PerformBiosHandoff();
// -----------------------------------------------------------------
// Step 5: Halt controller
// -----------------------------------------------------------------
uint32_t usbcmd = ReadOp(OP_USBCMD);
usbcmd &= ~USBCMD_RS;
WriteOp(OP_USBCMD, usbcmd);
// Wait for HCH (Halted) to be set
for (uint32_t i = 0; i < 100000; i++) {
if (ReadOp(OP_USBSTS) & USBSTS_HCH) {
break;
}
for (int j = 0; j < 10; j++) {
Arch::CompilerBarrier();
}
}
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) {
KernelLogStream(WARNING, "xHCI") << "Controller failed to halt";
}
KernelLogStream(OK, "xHCI") << "Controller halted";
// -----------------------------------------------------------------
// Step 6: Reset controller
// -----------------------------------------------------------------
WriteOp(OP_USBCMD, USBCMD_HCRST);
// Wait for HCRST to clear
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBCMD) & USBCMD_HCRST)) {
break;
}
for (int j = 0; j < 10; j++) {
Arch::CompilerBarrier();
}
}
// Wait for CNR (Controller Not Ready) to clear
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_CNR)) {
break;
}
for (int j = 0; j < 10; j++) {
Arch::CompilerBarrier();
}
}
if (ReadOp(OP_USBSTS) & USBSTS_CNR) {
KernelLogStream(WARNING, "xHCI") << "Controller not ready after reset";
}
KernelLogStream(OK, "xHCI") << "Controller reset complete";
// -----------------------------------------------------------------
// Step 7: Program CONFIG register (MaxSlotsEn)
// -----------------------------------------------------------------
WriteOp(OP_CONFIG, g_maxSlots);
// -----------------------------------------------------------------
// Step 8: Allocate DCBAA
// -----------------------------------------------------------------
g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
// Write DCBAAP (64-bit, split into two 32-bit writes)
WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
WriteOp(OP_DCBAAP + 4, (uint32_t)(g_dcbaaPhys >> 32));
KernelLogStream(OK, "xHCI") << "DCBAA at phys " << base::hex << g_dcbaaPhys;
// -----------------------------------------------------------------
// Step 9: Scratchpad buffers
// -----------------------------------------------------------------
if (maxScratchpadBufs > 0) {
uint64_t spArrayPhys;
g_scratchpadBufs = (uint64_t*)AllocateDmaBuffer(spArrayPhys);
for (uint32_t i = 0; i < maxScratchpadBufs; i++) {
uint64_t bufPhys;
AllocateDmaBuffer(bufPhys);
g_scratchpadBufs[i] = bufPhys;
}
// DCBAA[0] = physical address of the scratchpad buffer array
g_dcbaa[0] = spArrayPhys;
KernelLogStream(OK, "xHCI") << "Allocated " << base::dec << (uint64_t)maxScratchpadBufs << " scratchpad buffers";
}
// -----------------------------------------------------------------
// Step 10: Command ring
// -----------------------------------------------------------------
g_cmdRing = (TRB*)AllocateDmaBuffer(g_cmdRingPhys);
// Set up Link TRB at the last position
TRB& linkTrb = g_cmdRing[CMD_RING_SIZE - 1];
linkTrb.Parameter0 = (uint32_t)(g_cmdRingPhys & 0xFFFFFFFF);
linkTrb.Parameter1 = (uint32_t)(g_cmdRingPhys >> 32);
linkTrb.Status = 0;
linkTrb.Control = (TRB_LINK << TRB_TYPE_SHIFT) | TRB_ENT;
// Toggle Cycle bit is bit 1 in the Link TRB control - use TRB_ENT which is bit 1
// Write CRCR = command ring physical address | cycle bit 1
uint64_t crcr = g_cmdRingPhys | TRB_CYCLE_BIT;
WriteOp(OP_CRCR, (uint32_t)(crcr & 0xFFFFFFFF));
WriteOp(OP_CRCR + 4, (uint32_t)(crcr >> 32));
g_cmdRingCCS = true;
g_cmdRingEnqueue = 0;
KernelLogStream(OK, "xHCI") << "Command ring at phys " << base::hex << g_cmdRingPhys;
// -----------------------------------------------------------------
// Step 11: Event ring + ERST
// -----------------------------------------------------------------
g_evtRing = (TRB*)AllocateDmaBuffer(g_evtRingPhys);
g_erst = (ERSTEntry*)AllocateDmaBuffer(g_erstPhys);
// Set up ERST entry 0
g_erst[0].RingSegmentBase = g_evtRingPhys;
g_erst[0].RingSegmentSize = EVT_RING_SIZE;
g_erst[0].Reserved = 0;
// Write interrupter 0 registers
// Order: ERSTSZ → ERDP → ERSTBA (ERSTBA triggers hardware read of ERST)
WriteRt(IR0_ERSTSZ, 1);
WriteRt(IR0_ERDP, (uint32_t)(g_evtRingPhys & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(g_evtRingPhys >> 32));
// Write ERSTBA last (triggers hardware to read the ERST)
WriteRt(IR0_ERSTBA, (uint32_t)(g_erstPhys & 0xFFFFFFFF));
WriteRt(IR0_ERSTBA + 4, (uint32_t)(g_erstPhys >> 32));
g_evtRingCCS = true;
g_evtRingDequeue = 0;
KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
// -----------------------------------------------------------------
// Step 12: MSI setup
// -----------------------------------------------------------------
if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function) && !SetupIntx(foundDev->Bus, foundDev->Device, foundDev->Function)) {
KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
g_pollMode = true;
}
// -----------------------------------------------------------------
// Step 13: Enable interrupter 0
// -----------------------------------------------------------------
WriteRt(IR0_IMAN, IMAN_IE);
WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
// -----------------------------------------------------------------
// Step 14: Start controller
// -----------------------------------------------------------------
WriteOp(OP_USBCMD, USBCMD_RS | USBCMD_INTE | USBCMD_HSEE);
// Wait for controller to start (HCH should clear)
for (uint32_t i = 0; i < 100000; i++) {
if (!(ReadOp(OP_USBSTS) & USBSTS_HCH)) {
break;
}
for (int j = 0; j < 10; j++) {
Arch::CompilerBarrier();
}
}
KernelLogStream(OK, "xHCI") << "Controller started";
g_initialized = true;
// -----------------------------------------------------------------
// Step 14.5: Power on all ports
// -----------------------------------------------------------------
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (!(portsc & PORTSC_PP)) {
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE, PORTSC_PP);
}
}
// Wait for port power to stabilize (~20ms)
BusyWaitMs(20);
KernelLogStream(OK, "xHCI") << "All ports powered";
// -----------------------------------------------------------------
// Step 15: Port scanning
// -----------------------------------------------------------------
for (uint32_t port = 0; port < g_maxPorts; port++) {
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
// Check if device is connected (CCS)
if (!(portsc & PORTSC_CCS)) {
continue;
}
KernelLogStream(INFO, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": device connected, PORTSC=" << base::hex << (uint64_t)portsc;
// Reset the port: preserve power, clear change bits, set port reset
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_PR | PORTSC_CHANGE_BITS);
// Wait for Port Reset Change (PRC) to be set
bool resetDone = false;
for (uint32_t i = 0; i < 100000; i++) {
PollEvents();
uint32_t ps = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
if (ps & PORTSC_PRC) {
resetDone = true;
break;
}
for (int j = 0; j < 100; j++) {
Arch::CompilerBarrier();
}
}
if (!resetDone) {
KernelLogStream(WARNING, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1) << " reset timeout";
continue;
}
// Re-read PORTSC after reset
portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
uint32_t speed = (portsc >> 10) & 0xF;
// Clear change bits
WriteOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
const char* speedStr = "Unknown";
switch (speed) {
case SPEED_FULL: speedStr = "Full (12 Mbps)"; break;
case SPEED_LOW: speedStr = "Low (1.5 Mbps)"; break;
case SPEED_HIGH: speedStr = "High (480 Mbps)"; break;
case SPEED_SUPER: speedStr = "Super (5 Gbps)"; break;
}
KernelLogStream(OK, "xHCI") << "Port " << base::dec << (uint64_t)(port + 1)
<< ": reset complete, speed=" << speedStr;
// Post-reset recovery delay (USB spec requires >= 10ms)
BusyWaitMs(10);
// Enumerate the device (port IDs are 1-based)
UsbDevice::EnumerateDevice(port + 1, speed);
}
g_bootScanComplete = true;
KernelLogStream(OK, "xHCI") << "Initialization complete";
}
};
Reference in new issue View Git Blame Copy Permalink