daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: aarch64 port - Raspberry Pi 4/400 SD boot (SDHCI, ext2 root + first-boot grow), USB hubs, PCI INTx routing, SMP, DMA fences

Browse Source
This commit is contained in:
daniel committed 2026-10-25 13:48:26 +01:00
1 parent 8c09ccef66
commit 808e1d66fd
66 files changed
+3690 -196
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernel/src/Drivers/USB/Xhci.cpp
+385 -62
View File
@@ -11,11 +11,14 @@
#include "HidKeyboard.hpp"
#include "HidMouse.hpp"
#include "MassStorage.hpp"
#include "Hub.hpp"
#include "UserUsb.hpp"
#include <Pci/Pci.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/Dma.hpp>
#include <ACPI/GsiRouting.hpp>
#include <Memory/Paging.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
@@ -59,6 +62,21 @@ namespace Drivers::USB::Xhci {
static std::atomic<bool> g_hotplugProcessing{false};
static std::atomic<bool> g_irqDrainPending{false};
// HID interrupt endpoints that completed with an error. The event drain
// does not re-arm them (a halted endpoint takes commands to clear, which
// cannot be waited on from there): it records the error, and the
// deferred-work pass recovers the endpoint once g_hidRecoverDueMs is
// reached. Low/full-speed devices behind a hub's TT (every mouse on a
// Raspberry Pi 4) see transaction and split errors often enough that
// without this the device goes dead until replugged.
// g_hidErrorStreak counts errors since the last good report; it sets the
// backoff, and at HID_REENUMERATE_STREAK the device is reset and
// enumerated again, as a replug would.
static constexpr uint32_t HID_REENUMERATE_STREAK = 6;
static std::atomic<uint32_t> g_hidErrorCc[MAX_SLOTS + 1] = {}; // 0 = no recovery due
static std::atomic<uint32_t> g_hidErrorStreak[MAX_SLOTS + 1] = {};
static std::atomic<uint64_t> g_hidRecoverDueMs[MAX_SLOTS + 1] = {};
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
// a process-context mutex, so printing there can deadlock if the interrupt
// preempts a CPU that already owns the terminal lock. Capture the small
@@ -243,18 +261,44 @@ namespace Drivers::USB::Xhci {
return *(volatile uint32_t*)(g_mmioBase + reg);
}
// TRBs (and the buffers they point at) must be visible before the
// controller is told to look.
static void WriteDoorbell(uint32_t index, uint32_t value) {
Arch::DmaWriteFence();
*(volatile uint32_t*)(g_dbBase + index * 4) = value;
}
// Hand a TRB to the controller: everything else in it first, then the
// control word that carries its cycle bit. A running ring may fetch the
// TRB the moment the cycle bit matches.
static inline void PublishTrb(TRB& trb, uint32_t control) {
Arch::DmaWriteFence();
*(volatile uint32_t*)&trb.Control = control;
}
// -------------------------------------------------------------------------
// DMA buffer allocation
// -------------------------------------------------------------------------
// Whether the controller's DMA snoops the CPU caches. PCI controllers
// are taken to be coherent; a platform controller says so in _CCA.
static bool g_dmaCoherent = true;
uint8_t* AllocateDmaPage(uint64_t& outPhysAddr) {
return (uint8_t*)Memory::Dma::AllocatePage(outPhysAddr, g_dmaCoherent);
}
void* AllocateDmaPage() {
uint64_t phys;
return AllocateDmaPage(phys);
}
void FreeDmaPage(void* page) {
Memory::Dma::FreePage(page, g_dmaCoherent);
}
static uint8_t* AllocateDmaBuffer(uint64_t& outPhysAddr) {
void* virt = Memory::g_pfa->AllocateZeroed();
outPhysAddr = Memory::SubHHDM(virt);
return (uint8_t*)virt;
return AllocateDmaPage(outPhysAddr);
}
// -------------------------------------------------------------------------
@@ -419,6 +463,24 @@ namespace Drivers::USB::Xhci {
// MSI setup (same pattern as E1000E)
// -------------------------------------------------------------------------
// Legacy INTx, for controllers without MSI (or platforms that cannot
// receive it): the line the firmware or Pci::AssignInterruptLines put
// in the Interrupt Line register. HandleInterrupt suits a level line:
// clearing IMAN.IP drops it.
static bool SetupIntx(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t line = Pci::LegacyRead8(bus, dev, func, (uint8_t)Pci::PCI_REG_INTERRUPT);
if (line == 0 || line >= 24) return false;
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND,
pciCmd & ~Pci::PCI_CMD_INTX_DISABLE);
Arch::RegisterIrqHandler(line, HandleInterrupt);
Arch::EnableLegacyIrq(line);
KernelLogStream(OK, "xHCI") << "Using INTx: platform line " << base::dec << (uint64_t)line;
return true;
}
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
@@ -523,6 +585,8 @@ namespace Drivers::USB::Xhci {
if (evtCycle != g_evtRingCCS) {
break; // No more events
}
// The rest of the event was written before its cycle bit.
Arch::DmaReadFence();
uint32_t trbType = (evt.Control & TRB_TYPE_MASK) >> TRB_TYPE_SHIFT;
lastType = trbType;
@@ -545,8 +609,12 @@ namespace Drivers::USB::Xhci {
WriteOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE,
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
// Defer enumeration to ProcessDeferredWork outside interrupt context.
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
// Defer enumeration to ProcessDeferredWork outside interrupt
// context. Changes during the boot scan are kept too (that
// pass skips ports already enumerated): a device that
// connects after the scan looked at its port, slower on
// real hardware than in QEMU, would otherwise be lost.
if (portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
@@ -642,12 +710,17 @@ namespace Drivers::USB::Xhci {
uint16_t len = (residual < reqLen)
? (uint16_t)(reqLen - residual) : 0;
if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
// Re-armed once the change is handled.
Hub::OnStatusChange((uint8_t)slotId, g_interruptDataBuf[slotId],
len, completionCode);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
HidKeyboard::ProcessReport(g_interruptDataBuf[slotId], len);
} else if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
HidMouse::ProcessReport(g_interruptDataBuf[slotId], len);
}
g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
// Re-queue for next HID report
QueueInterruptTransfer(slotId);
} else if (g_transferCallbacks[slotId]) {
@@ -672,6 +745,23 @@ namespace Drivers::USB::Xhci {
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
if (dev.InterfaceClass == UsbDevice::CLASS_HUB && dev.InterruptEpNum &&
epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
Hub::OnStatusChange((uint8_t)slotId, nullptr, 0, completionCode);
}
if (dev.InterfaceClass == UsbDevice::CLASS_HID && dev.InterruptEpNum &&
epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
// Retry at once, then back off 10 ms doubling.
uint32_t streak = g_hidErrorStreak[slotId].fetch_add(
1, std::memory_order_relaxed) + 1;
uint64_t delay = streak <= 1 ? 0 : 10ull << (streak - 2);
if (delay > 1000) delay = 1000;
g_hidRecoverDueMs[slotId].store(
Timekeeping::GetMilliseconds() + delay, std::memory_order_relaxed);
g_hidErrorCc[slotId].store(completionCode, std::memory_order_release);
}
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
g_transferCallbacks[slotId](slotId, epDci,
@@ -787,7 +877,7 @@ namespace Drivers::USB::Xhci {
if (g_cmdRingCCS) {
control |= TRB_CYCLE_BIT;
}
slot.Control = control;
PublishTrb(slot, control);
// Advance enqueue pointer
g_cmdRingEnqueue++;
@@ -836,10 +926,21 @@ namespace Drivers::USB::Xhci {
// ControlTransfer - perform a control transfer on EP0
// -------------------------------------------------------------------------
static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, uint64_t dataPhys, bool dirIn, bool trueNested);
// Data stages of a non-coherent controller go through this page (taken
// under g_controlXferLock). Callers pass small stack buffers, and cache
// maintenance cannot make those safe: a cache line the buffer shares with
// live stack data that the CPU dirties while the transfer runs is written
// back over what the device wrote.
static uint8_t* g_ctrlBounce = nullptr;
static uint64_t g_ctrlBouncePhys = 0;
uint32_t ControlTransfer(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, bool dirIn) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
return 0xFF;
}
@@ -853,6 +954,32 @@ namespace Drivers::USB::Xhci {
g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
if (!trueNested) g_controlXferLock.Acquire();
bool hasData = data != nullptr && wLength != 0;
bool bounce = hasData && !g_dmaCoherent && !trueNested &&
g_ctrlBounce != nullptr && wLength <= 0x1000;
uint32_t cc;
if (bounce) {
// Copied in both directions: bytes a short IN transfer leaves
// alone keep the caller's contents, as with a direct transfer.
memcpy(g_ctrlBounce, data, wLength);
cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex,
wLength, g_ctrlBounce, g_ctrlBouncePhys, dirIn, false);
if (dirIn) memcpy(data, g_ctrlBounce, wLength);
} else {
// The data stage buffer is the caller's (cacheable) memory.
if (hasData) Memory::Dma::SyncForDevice(data, wLength, g_dmaCoherent);
cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex, wLength,
data, hasData ? Memory::SubHHDM(data) : 0, dirIn, trueNested);
if (hasData && dirIn) Memory::Dma::SyncForCpu(data, wLength, g_dmaCoherent);
}
if (!trueNested) g_controlXferLock.Release();
return cc;
}
static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
void* data, uint64_t dataPhys, bool dirIn, bool trueNested) {
UsbDeviceInfo& dev = g_devices[slotId];
// --- Setup Stage TRB ---
@@ -870,15 +997,15 @@ namespace Drivers::USB::Xhci {
if (dev.EP0RingCCS) {
setupControl |= TRB_CYCLE_BIT;
}
setup.Control = setupControl;
// The Setup TRB's cycle bit is published only after the Data and
// Status TRBs are complete (below): if EP0 is still running, it
// would otherwise run into a half-built TD.
// Advance EP0 enqueue (handles Link TRB wrap)
AdvanceEP0Ring(dev);
// --- Data Stage TRB (if wLength > 0) ---
if (wLength > 0 && data != nullptr) {
uint64_t dataPhys = Memory::SubHHDM(data);
TRB& dataTrb = dev.EP0Ring[dev.EP0RingEnqueue];
dataTrb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
dataTrb.Parameter1 = (uint32_t)(dataPhys >> 32);
@@ -891,7 +1018,7 @@ namespace Drivers::USB::Xhci {
if (dev.EP0RingCCS) {
dataControl |= TRB_CYCLE_BIT;
}
dataTrb.Control = dataControl;
PublishTrb(dataTrb, dataControl);
AdvanceEP0Ring(dev);
}
@@ -912,10 +1039,12 @@ namespace Drivers::USB::Xhci {
if (dev.EP0RingCCS) {
statusControl |= TRB_CYCLE_BIT;
}
status.Control = statusControl;
PublishTrb(status, statusControl);
AdvanceEP0Ring(dev);
PublishTrb(setup, setupControl);
// Ring doorbell for this slot, target EP0 (DCI 1)
g_xferCompleted[slotId] = false;
WriteDoorbell(slotId, 1);
@@ -939,7 +1068,6 @@ namespace Drivers::USB::Xhci {
while (Timekeeping::GetMilliseconds() - xferStart < 2000) {
PollEvents();
if (g_xferCompleted[slotId]) {
g_controlXferLock.Release();
return g_xferCompletionCode[slotId];
}
for (int j = 0; j < 100; j++) {
@@ -973,7 +1101,6 @@ namespace Drivers::USB::Xhci {
| (1 << 16); // DCI=1 (EP0)
SendCommand(deqTrb);
g_controlXferLock.Release();
return 0xFF;
}
@@ -1003,7 +1130,7 @@ namespace Drivers::USB::Xhci {
if (dev.InterruptRingCCS) {
control |= TRB_CYCLE_BIT;
}
trb.Control = control;
PublishTrb(trb, control);
// Advance enqueue (handles Link TRB wrap)
AdvanceInterruptRing(dev);
@@ -1140,7 +1267,7 @@ namespace Drivers::USB::Xhci {
if (dev.BulkInRingCCS) {
control |= TRB_CYCLE_BIT;
}
trb.Control = control;
PublishTrb(trb, control);
AdvanceBulkInRing(dev);
@@ -1267,6 +1394,9 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
void QueueBulkOutTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
if (data != nullptr && length != 0) {
Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
}
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
UsbDeviceInfo& dev = g_devices[slotId];
@@ -1281,7 +1411,7 @@ namespace Drivers::USB::Xhci {
if (dev.BulkOutRingCCS) {
control |= TRB_CYCLE_BIT;
}
trb.Control = control;
PublishTrb(trb, control);
AdvanceBulkOutRing(dev);
@@ -1294,8 +1424,23 @@ namespace Drivers::USB::Xhci {
// BulkTransfer - blocking bulk IN/OUT transfer
// -------------------------------------------------------------------------
static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength);
uint32_t BulkTransfer(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength) {
if (data != nullptr && length != 0) {
Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
}
uint32_t cc = BulkTransferUnsynced(slotId, dirIn, data, dataPhys, length, actualLength);
if (data != nullptr && length != 0 && dirIn) {
Memory::Dma::SyncForCpu(data, length, g_dmaCoherent);
}
return cc;
}
static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
uint32_t length, uint32_t* actualLength) {
if (actualLength) *actualLength = 0;
if (length == 0) return CC_SUCCESS;
if (length > 0x10000) return 0xFF;
@@ -1318,7 +1463,7 @@ namespace Drivers::USB::Xhci {
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
if (dev.BulkInRingCCS) control |= TRB_CYCLE_BIT;
trb.Control = control;
PublishTrb(trb, control);
AdvanceBulkInRing(dev);
target = dev.BulkInEpNum * 2 + 1;
@@ -1332,7 +1477,7 @@ namespace Drivers::USB::Xhci {
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
if (dev.BulkOutRingCCS) control |= TRB_CYCLE_BIT;
trb.Control = control;
PublishTrb(trb, control);
AdvanceBulkOutRing(dev);
target = dev.BulkOutEpNum * 2;
@@ -1412,6 +1557,17 @@ namespace Drivers::USB::Xhci {
return g_initialized;
}
// A HID endpoint whose error backoff has run out.
static bool HidRecoveryDue() {
uint64_t now = 0;
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
if (now == 0) now = Timekeeping::GetMilliseconds();
if (now >= g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) return true;
}
return false;
}
bool HasDeferredWork() {
if (g_initialized && g_pollMode && EventRingHasWork()) {
g_irqDrainPending.store(true, std::memory_order_release);
@@ -1419,6 +1575,7 @@ namespace Drivers::USB::Xhci {
return g_initialized &&
(g_irqDrainPending.load(std::memory_order_acquire) ||
g_deferredWorkPending.load(std::memory_order_acquire) ||
(g_bootScanComplete && (Hub::HasPendingWork() || HidRecoveryDue())) ||
g_diagnosticWorkPending.load(std::memory_order_acquire));
}
@@ -1468,7 +1625,9 @@ namespace Drivers::USB::Xhci {
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
Hub::UnregisterHub(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
MassStorage::UnregisterDevice(slotId);
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
@@ -1491,6 +1650,105 @@ namespace Drivers::USB::Xhci {
}
}
void RemoveDevice(uint8_t slotId) {
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
// Children first: a hub's slot outlives the devices addressed through it.
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (s != slotId && g_devices[s].Active && g_devices[s].ParentSlot == slotId) {
RemoveDevice(s);
}
}
UserUsb::DeviceDisconnected(slotId);
UnregisterClassDriver(slotId, g_devices[slotId]);
g_devices[slotId].Active = false;
g_hidErrorCc[slotId].store(0, std::memory_order_relaxed);
g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
g_transferCallbacks[slotId] = nullptr;
DisableSlot(slotId);
g_dcbaa[slotId] = 0;
g_devices[slotId] = {};
KernelLogStream(INFO, "xHCI") << "Slot " << base::dec << (uint64_t)slotId << " deactivated";
}
// Bring a HID device's interrupt endpoint back after a failed transfer.
// Reset Endpoint clears a halt (a Context State Error when the error did
// not halt it), Set TR Dequeue skips the failed TRB, and CLEAR_FEATURE
// (ENDPOINT_HALT) clears a stall on the device and puts its data toggle
// back to DATA0 to match the reset host side (USB 2.0 9.4.5). If errors keep coming, the
// device is removed and its port reset, as a replug would.
static void RecoverHidEndpoint(uint8_t slotId, uint32_t cc) {
UsbDeviceInfo& dev = g_devices[slotId];
if (!dev.Active || dev.InterfaceClass != UsbDevice::CLASS_HID ||
dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
uint32_t streak = g_hidErrorStreak[slotId].load(std::memory_order_relaxed);
if (streak >= HID_REENUMERATE_STREAK) {
KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
<< ": HID endpoint keeps failing (cc=" << (uint64_t)cc
<< "), resetting the device";
uint8_t parentSlot = dev.ParentSlot, parentPort = dev.ParentPort;
uint8_t rootPort = dev.PortId;
RemoveDevice(slotId);
if (parentSlot != 0) {
Hub::RequestPortCheck(parentSlot, parentPort);
} else if (rootPort >= 1 && rootPort <= g_maxPorts) {
// Disable the port (PED is write-1-to-clear) so the hot-plug
// pass resets it before enumerating.
uint32_t reg = OP_PORTSC_BASE + (rootPort - 1) * OP_PORTSC_STRIDE;
WriteOp(reg, (ReadOp(reg) & PORTSC_PRESERVE) | PORTSC_PED);
g_hotplugPending[rootPort - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
return;
}
if (streak <= 1) {
KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
<< ": HID transfer failed (cc=" << (uint64_t)cc << "), recovering endpoint";
}
uint8_t dci = dev.InterruptEpNum * 2 + 1;
TRB resetTrb = {};
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
uint64_t newDeq = dev.InterruptRingPhys
+ (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
if (dev.InterruptRingCCS) newDeq |= 1; // DCS bit
TRB deqTrb = {};
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
| ((uint32_t)slotId << 24)
| ((uint32_t)dci << 16);
SendCommand(deqTrb);
// Only after a real reset: the host toggle is untouched otherwise.
if (wasHalted) {
ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
(uint16_t)(0x80 | dev.InterruptEpNum), 0, nullptr, false);
}
if (dev.Active) QueueInterruptTransfer(slotId);
}
static void ProcessHidRecovery() {
uint64_t now = Timekeeping::GetMilliseconds();
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
if (now < g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) continue;
uint32_t cc = g_hidErrorCc[s].exchange(0, std::memory_order_acq_rel);
if (cc != 0) RecoverHidEndpoint(s, cc);
}
}
// -------------------------------------------------------------------------
// ProcessDeferredWork - handle hot-plug outside interrupt context.
// Called from the BSP idle path so class-driver probing can wait on timers.
@@ -1508,7 +1766,8 @@ namespace Drivers::USB::Xhci {
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
bool haveHotplug = g_bootScanComplete &&
g_deferredWorkPending.load(std::memory_order_acquire);
(g_deferredWorkPending.load(std::memory_order_acquire) || Hub::HasPendingWork() ||
HidRecoveryDue());
if (!haveDiagnostics && !haveHotplug) return;
// CAS claim: any idle core may call this, but only one runs at a time.
@@ -1525,6 +1784,9 @@ namespace Drivers::USB::Xhci {
return;
}
// First: a device it gives up on is queued for the hot-plug pass below.
ProcessHidRecovery();
g_deferredWorkPending.store(false, std::memory_order_release);
for (uint32_t port = 0; port < g_maxPorts; port++) {
@@ -1585,25 +1847,23 @@ namespace Drivers::USB::Xhci {
UsbDevice::EnumerateDevice(port + 1, speed);
}
} else {
// Device disconnected — deactivate its slot
// Device disconnected: deactivate its slot (and, for a hub,
// the slots of everything behind it)
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
UserUsb::DeviceDisconnected(s);
UnregisterClassDriver(s, g_devices[s]);
g_devices[s].Active = false;
g_transferCallbacks[s] = nullptr;
DisableSlot(s);
g_dcbaa[s] = 0;
g_devices[s] = {};
KernelLogStream(INFO, "xHCI") << "Hot-unplug: slot "
<< base::dec << (uint64_t)s << " (port "
<< (uint64_t)(port + 1) << ") deactivated";
if (g_devices[s].Active && g_devices[s].PortId == port + 1 &&
g_devices[s].ParentSlot == 0) {
KernelLogStream(INFO, "xHCI") << "Hot-unplug: port "
<< base::dec << (uint64_t)(port + 1);
RemoveDevice(s);
break;
}
}
}
}
// Port changes behind hubs.
Hub::ProcessPendingWork();
g_hotplugProcessing.store(false, std::memory_order_release);
}
@@ -1611,32 +1871,22 @@ namespace Drivers::USB::Xhci {
// Probe (called by PCI driver matching framework)
// -------------------------------------------------------------------------
bool Probe(const Pci::PciDevice& dev) {
if (g_initialized) return false;
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
<< base::hex << (uint64_t)dev.Bus << ":"
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
// Read BAR0 and map MMIO region
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
constexpr uint64_t MmioSize = 0x10000;
for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
// Map the register block, reset the controller and set up its
// context and command/event rings. Interrupt delivery is the caller's.
static void InitializeController(uint64_t mmioPhys, uint64_t mmioSize) {
for (uint64_t offset = 0; offset < mmioSize; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
// Parse capability registers. CAPLENGTH and HCIVERSION share the
// first dword; read it whole, since narrower accesses are not safe
// behind every PCIe bridge (Linux does the same).
uint32_t capBase = ReadCap(CAP_CAPLENGTH);
g_capLength = (uint8_t)(capBase & 0xFF);
// Parse capability registers
g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
uint32_t hciVersion = capBase >> 16;
KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
<< ", CapLength: " << (uint64_t)g_capLength;
@@ -1656,6 +1906,16 @@ namespace Drivers::USB::Xhci {
g_rtBase = g_mmioBase + rtsOff;
g_dbBase = g_mmioBase + dbOff;
// A firmware-described register block may be smaller than what the
// capability registers point into (port registers, interrupter 0,
// the doorbells): map those too.
uint64_t needed = g_capLength + OP_PORTSC_BASE + (uint64_t)((hcsParams1 >> 24) & 0xFF) * OP_PORTSC_STRIDE;
if (rtsOff + 0x40 > needed) needed = rtsOff + 0x40;
if (dbOff + 4 * ((hcsParams1 & 0xFF) + 1) > needed) needed = dbOff + 4 * ((hcsParams1 & 0xFF) + 1);
for (uint64_t offset = mmioSize; offset < needed; offset += 0x1000) {
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
}
KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
<< ", MaxPorts: " << (uint64_t)g_maxPorts
<< ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
@@ -1701,6 +1961,8 @@ namespace Drivers::USB::Xhci {
// Program CONFIG
WriteOp(OP_CONFIG, g_maxSlots);
if (!g_dmaCoherent) g_ctrlBounce = AllocateDmaBuffer(g_ctrlBouncePhys);
// Allocate DCBAA
g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
@@ -1748,13 +2010,10 @@ namespace Drivers::USB::Xhci {
g_evtRingCCS = true;
g_evtRingDequeue = 0;
KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
}
// MSI setup
if (!SetupMsi(dev.Bus, dev.Device, dev.Function)) {
KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
g_pollMode = true;
}
// Enable the interrupter, run the controller, power and enumerate ports.
static bool StartController() {
// Enable interrupter 0
WriteRt(IR0_IMAN, IMAN_IE);
WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
@@ -1822,11 +2081,75 @@ namespace Drivers::USB::Xhci {
UsbDevice::EnumerateDevice(port + 1, speed);
}
// Connected ports the scan found nothing on (still training, or the
// enumeration failed) get another look from the hot-plug path.
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!(ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE) & PORTSC_CCS)) continue;
bool active = false;
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
if (g_devices[s].Active && g_devices[s].PortId == port + 1) active = true;
}
if (!active) {
g_hotplugPending[port].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
}
g_bootScanComplete = true;
KernelLogStream(OK, "xHCI") << "Initialization complete";
return true;
}
bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
bool activeLow, bool dmaCoherent) {
if (g_initialized) return false;
g_dmaCoherent = dmaCoherent;
InitializeController(mmioPhys, mmioSize < 0x1000 ? 0x1000 : (mmioSize + 0xFFF) & ~0xFFFULL);
uint8_t line = gsi != 0 ? Hal::GsiRouting::LineForGsi(gsi, levelTriggered, activeLow) : 0;
if (line != 0) {
Arch::RegisterIrqHandler(line, HandleInterrupt);
Arch::EnableLegacyIrq(line);
KernelLogStream(OK, "xHCI") << "Interrupt: GSI " << base::dec << (uint64_t)gsi
<< " on platform line " << (uint64_t)line;
} else {
KernelLogStream(WARNING, "xHCI") << "No interrupt, using poll mode";
}
// Watch the event ring from the timer as well. The interrupt is a
// PCI INTx routed by firmware through a bridge the kernel does not
// drive; if it never arrives, input still works (a ring read per
// tick costs next to nothing).
g_pollMode = true;
return StartController();
}
bool Probe(const Pci::PciDevice& dev) {
if (g_initialized) return false;
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
<< base::hex << (uint64_t)dev.Bus << ":"
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
InitializeController(mmioPhys, 0x10000);
// MSI setup
if (!SetupMsi(dev.Bus, dev.Device, dev.Function) && !SetupIntx(dev.Bus, dev.Device, dev.Function)) {
KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
g_pollMode = true;
}
return StartController();
}
// -------------------------------------------------------------------------
// Initialize (standalone fallback path)
// -------------------------------------------------------------------------
@@ -2064,8 +2387,8 @@ namespace Drivers::USB::Xhci {
// -----------------------------------------------------------------
// Step 12: MSI setup
// -----------------------------------------------------------------
if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function)) {
KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function) && !SetupIntx(foundDev->Bus, foundDev->Device, foundDev->Function)) {
KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
g_pollMode = true;
}