daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: kernel concurrency, interrupt context, and user VA safety

Browse Source
This commit is contained in:
daniel committed 2026-08-01 15:43:38 +02:00
1 parent f222bf1f04
commit b1c55073c7
71 files changed
+2654 -1092
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernel/src/Drivers/USB/Bluetooth/Avrcp.cpp
+6 -1
View File
@@ -47,7 +47,12 @@ namespace Drivers::USB::Bluetooth::Avrcp {
// AVRCP notification events
constexpr uint8_t EVT_VOLUME_CHANGED = 0x0D;
static kcp::Spinlock g_notifyLock;
// AVRCP callbacks are dispatched from xHCI's deferred PollEvents path, not
// from the hard IRQ. Sending the INTERIM/CHANGED response can descend into
// the HCI and xHCI transmit rings, so keep interrupts enabled while this
// state is serialized. An IRQ-disabling spinlock here can stall the local
// timer behind an unrelated transmit/recovery operation.
static kcp::Mutex g_notifyLock;
static uint16_t g_volumeNotifyCid = 0;
static uint8_t g_volumeNotifyTransaction = 0;
kernel/src/Drivers/USB/Bluetooth/Bluetooth.cpp
+2 -1
View File
@@ -406,12 +406,13 @@ namespace Drivers::USB::Bluetooth {
// system this briefly pauses userspace, matching the old synchronous
// behavior minus the boot-path stall.
auto* cpu = Smp::GetCurrentCpuData();
bool wasReserved = cpu && cpu->reservedForKernelWork;
if (cpu) cpu->reservedForKernelWork = true;
Hci::SetFwTrace(true); // bounded per-completion event-pipe trace
CompleteInit();
Hci::DumpFwTrace(); // flush remaining records (process context)
Hci::SetFwTrace(false);
if (cpu) cpu->reservedForKernelWork = false;
if (cpu) cpu->reservedForKernelWork = wasReserved;
}
// =========================================================================
kernel/src/Drivers/USB/Bluetooth/Hci.cpp
+14 -8
View File
@@ -59,10 +59,9 @@ namespace Drivers::USB::Bluetooth::Hci {
//
// The trace is recorded into a lock-free ring and printed LATER by
// DumpFwTrace() from process context. It must NEVER KernelLogStream from
// TransferCallback: that runs nested under PollEvents in xHCI MSI (IRQ)
// context, and the terminal lock is a non-IRQ-disabling Mutex -- an IRQ
// logger interrupting a same-core holder spins forever (observed: hard
// boot freeze right after "Intel version raw", 2026-07-06).
// TransferCallback: it runs nested under PollEvents while the event-ring
// ownership guard is held. Logging there can recursively pump or contend
// with unrelated subsystem locks, so keep it a bounded data-only callback.
static std::atomic<bool> g_fwTrace{false};
static std::atomic<uint32_t> g_fwTraceCount{0};
static std::atomic<uint32_t> g_intInCompletions{0};
@@ -148,8 +147,10 @@ namespace Drivers::USB::Bluetooth::Hci {
// guards, so one core can be sending a media packet while another sends a
// signaling reply; without this lock the two race on g_aclTxSlot and on
// the bulk OUT TRB ring's enqueue/cycle state, garbling packets on air.
// IRQ-disabling spinlock so the holder also can't be preempted mid-send.
static kcp::Spinlock g_aclTxLock;
// These paths run only from deferred/process context. Keep interrupts
// enabled: endpoint recovery also takes this lock and can wait for bounded
// xHCI commands, which must never happen below an IRQ-disabling spinlock.
static kcp::Mutex g_aclTxLock;
// HCI command DMA buffer (separate from ACL to avoid conflicts)
static uint8_t* g_cmdDmaBuf = nullptr;
@@ -559,8 +560,8 @@ namespace Drivers::USB::Bluetooth::Hci {
// Bounded bring-up trace: one RING RECORD per interrupt-IN
// completion while the firmware phase runs; DumpFwTrace() prints
// them later from process context. No logging here -- this runs
// in xHCI MSI (IRQ) context (see the deadlock note at the ring).
// them later from top-level process/idle context. No logging here:
// this callback is nested under the xHCI event-ring owner.
if (g_fwTrace.load(std::memory_order_relaxed)) {
uint32_t n = g_fwTraceCount.fetch_add(1, std::memory_order_relaxed);
if (n < FW_TRACE_CAP) {
@@ -2074,6 +2075,10 @@ namespace Drivers::USB::Bluetooth::Hci {
// all TRBs queued behind it. Free those DMA slots and remove only
// those never-delivered packets from the controller-credit count;
// earlier successful OUT transfers still await their NOCP events.
// Serialize the dequeue rewrite with SendAcl/firmware bulk enqueue.
// Without this, another CPU can publish a TRB between Reset Endpoint
// and Set TR Dequeue and have recovery silently discard or split it.
g_aclTxLock.Acquire();
uint32_t discarded = AclTxInFlight();
KernelLogStream(WARNING, "BT-HCI")
<< "Recovering HCI ACL transmit pipe; discarding "
@@ -2086,6 +2091,7 @@ namespace Drivers::USB::Bluetooth::Hci {
- (int32_t)discarded;
if (remaining < 0)
g_aclPendingCount.store(0, std::memory_order_release);
g_aclTxLock.Release();
}
// Asynchronous HCI events are processed at top level. Besides avoiding
kernel/src/Drivers/USB/HidMouse.cpp
+30 -10
View File
@@ -172,18 +172,23 @@ namespace Drivers::USB::HidMouse {
// -------------------------------------------------------------------------
static int32_t ExtractSigned(const uint8_t* data, uint16_t bitOffset, uint8_t bitSize) {
int32_t value = 0;
uint32_t value = 0;
for (uint8_t i = 0; i < bitSize; i++) {
uint16_t byteIdx = (bitOffset + i) / 8;
uint8_t bitIdx = (bitOffset + i) % 8;
if (data[byteIdx] & (1 << bitIdx))
value |= (1 << i);
value |= (1u << i);
}
// Sign extend
if (bitSize < 32 && (value & (1 << (bitSize - 1)))) {
value |= ~((1 << bitSize) - 1);
if (bitSize < 32 && (value & (1u << (bitSize - 1)))) {
value |= ~((1u << bitSize) - 1);
}
return value;
return (int32_t)value;
}
static bool FieldFits(uint32_t bitOffset, uint32_t bitSize, uint32_t reportBits) {
return bitSize > 0 && bitSize <= 32 && bitOffset <= reportBits
&& bitSize <= reportBits - bitOffset;
}
// -------------------------------------------------------------------------
@@ -222,18 +227,30 @@ namespace Drivers::USB::HidMouse {
// Skip report ID byte if present
uint16_t byteOffset = g_Format.hasReportId ? 8 : 0;
if (g_Format.hasReportId && data[0] != g_Format.reportId) return;
uint32_t reportBits = (uint32_t)length * 8;
uint32_t buttonBits = g_Format.buttonCount < 8 ? g_Format.buttonCount : 8;
uint32_t buttonOffset = byteOffset + g_Format.buttonBitOffset;
uint32_t xOffset = byteOffset + g_Format.xBitOffset;
uint32_t yOffset = byteOffset + g_Format.yBitOffset;
if ((buttonBits != 0 && !FieldFits(buttonOffset, buttonBits, reportBits))
|| !FieldFits(xOffset, g_Format.xBitSize, reportBits)
|| !FieldFits(yOffset, g_Format.yBitSize, reportBits)) {
return;
}
// Extract buttons
uint8_t buttons = 0;
for (uint8_t i = 0; i < g_Format.buttonCount && i < 8; i++) {
uint16_t bit = byteOffset + g_Format.buttonBitOffset + i;
for (uint8_t i = 0; i < buttonBits; i++) {
uint16_t bit = (uint16_t)(buttonOffset + i);
if (data[bit / 8] & (1 << (bit % 8)))
buttons |= (1 << i);
}
// Extract X, Y
int32_t rawX = ExtractSigned(data, byteOffset + g_Format.xBitOffset, g_Format.xBitSize);
int32_t rawY = ExtractSigned(data, byteOffset + g_Format.yBitOffset, g_Format.yBitSize);
int32_t rawX = ExtractSigned(data, (uint16_t)xOffset, g_Format.xBitSize);
int32_t rawY = ExtractSigned(data, (uint16_t)yOffset, g_Format.yBitSize);
// Clamp to int8_t range for InjectMouseReport
if (rawX > 127) rawX = 127;
@@ -244,7 +261,10 @@ namespace Drivers::USB::HidMouse {
// Extract scroll wheel
int8_t scroll = 0;
if (g_Format.scrollBitSize > 0) {
int32_t rawScroll = ExtractSigned(data, byteOffset + g_Format.scrollBitOffset, g_Format.scrollBitSize);
uint32_t scrollOffset = byteOffset + g_Format.scrollBitOffset;
if (!FieldFits(scrollOffset, g_Format.scrollBitSize, reportBits)) return;
int32_t rawScroll = ExtractSigned(data, (uint16_t)scrollOffset,
g_Format.scrollBitSize);
if (rawScroll > 127) rawScroll = 127;
if (rawScroll < -128) rawScroll = -128;
scroll = (int8_t)rawScroll;
kernel/src/Drivers/USB/Xhci.cpp
+200 -53
View File
@@ -54,9 +54,30 @@ namespace Drivers::USB::Xhci {
static bool g_bootScanComplete = false; // true after initial port scan finishes
// Hot-plug deferred work
static volatile bool g_hotplugPending[MAX_PORTS] = {};
static volatile bool g_deferredWorkPending = false;
static std::atomic<bool> g_hotplugPending[MAX_PORTS] = {};
static std::atomic<bool> g_deferredWorkPending{false};
static std::atomic<bool> g_diagnosticWorkPending{false};
static std::atomic<bool> g_hotplugProcessing{false};
static std::atomic<bool> g_irqDrainPending{false};
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
// a process-context mutex, so printing there can deadlock if the interrupt
// preempts a CPU that already owns the terminal lock. Capture the small
// diagnostic payload atomically and print it from ProcessDeferredWork().
static std::atomic<uint32_t> g_transferErrorsPending{0};
static std::atomic<uint32_t> g_transferErrorReports{0};
static std::atomic<uint32_t> g_lastErrorSlot{0};
static std::atomic<uint32_t> g_lastErrorEp{0};
static std::atomic<uint32_t> g_lastErrorCc{0};
static std::atomic<uint32_t> g_stormsPending{0};
static std::atomic<uint32_t> g_stormReports{0};
static std::atomic<uint32_t> g_lastStormProcessed{0};
static std::atomic<uint32_t> g_lastStormPortEvents{0};
static std::atomic<uint32_t> g_lastStormTransferEvents{0};
static std::atomic<uint32_t> g_lastStormType{0};
static std::atomic<uint32_t> g_lastStormSlot{0};
static std::atomic<uint32_t> g_lastStormEp{0};
static std::atomic<uint32_t> g_lastStormCc{0};
// MMIO region pointers
static volatile uint8_t* g_mmioBase = nullptr;
@@ -78,6 +99,12 @@ namespace Drivers::USB::Xhci {
static uint64_t g_cmdRingPhys = 0;
static uint32_t g_cmdRingEnqueue = 0;
static bool g_cmdRingCCS = true;
// The controller exposes one command ring and one command-completion
// mailbox. Endpoint recovery, hot-plug enumeration, and control-transfer
// recovery can originate on different CPUs, so the whole submit/wait
// transaction must be single-owner. This is a process/idle-context lock;
// SendCommand is never called from a transfer-event callback.
static kcp::Mutex g_commandLock;
// Event ring
static TRB* g_evtRing = nullptr;
@@ -125,12 +152,12 @@ namespace Drivers::USB::Xhci {
// completion. Used by InPollContext() / ControlTransfer().
//
// MUST be a real atomic claimed with compare-exchange, not a plain
// volatile bool: PollEvents runs from the MSI handler (BSP) AND from
// syscall-context pumps (A2DP WriteAudio/StartSource on whatever core the
// app runs on) AND from idle cores (ProcessDeferredWork). A check-then-set
// on a volatile is a cross-core TOCTOU: two cores both pass the check,
// both drain the shared event ring, g_evtRingDequeue/g_evtRingCCS desync,
// and completions get double-delivered (duplicate ACL packets) or lost
// volatile bool: PollEvents runs from syscall-context pumps (A2DP
// WriteAudio/StartSource on whatever core the app runs on) and from idle
// cores (ProcessDeferredWork). A check-then-set on a volatile is a
// cross-core TOCTOU: two cores both pass the check, both drain the shared
// event ring, g_evtRingDequeue/g_evtRingCCS desync, and completions get
// double-delivered (duplicate ACL packets) or lost
// (missed AVDTP responses) -- both observed on HW once audio streaming
// started pumping from syscall context.
static std::atomic<bool> g_pollActive{false};
@@ -293,7 +320,7 @@ namespace Drivers::USB::Xhci {
// Forward declarations
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq);
static void HandleInterrupt(uint8_t irq, bool fromUser);
// -------------------------------------------------------------------------
// BIOS-to-OS handoff (USB Legacy Support)
@@ -389,10 +416,13 @@ namespace Drivers::USB::Xhci {
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability not found";
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
@@ -448,33 +478,37 @@ namespace Drivers::USB::Xhci {
// PollEvents - process event ring
// -------------------------------------------------------------------------
void PollEvents() {
// PollEvents runs both from the synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and from the xHCI MSI handler. On a
// single core the IRQ can preempt a poll loop mid-drain; if both advance
// g_evtRingDequeue / g_evtRingCCS the ring tracking desyncs and the
// cycle-bit check can start matching stale entries forever -> the boot
// freezes (observed wedging the Bluetooth firmware download at ~635 KB,
// where the dying device floods the event ring). Guard against re-entry:
// the interrupt is already acked (IMAN.IP cleared in HandleInterrupt) and
// the active poll loop drains these events itself. g_pollActive is also
// read by InPollContext() so command submitters (ControlTransfer) can
// tell they are nested and must fire-and-forget instead of waiting.
enum class PollResult : uint8_t {
Drained,
Busy,
BudgetExhausted,
};
static PollResult PollEventsBounded(uint32_t maxEvents) {
// PollEvents runs from synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and idle bottom halves on different
// CPUs. If two contexts advance g_evtRingDequeue / g_evtRingCCS, ring
// tracking desyncs and the cycle-bit check can start matching stale
// entries forever -> the boot freezes (observed wedging the Bluetooth
// firmware download at ~635 KB, where the dying device floods the event
// ring). Guard against cross-core re-entry. g_pollActive is also read by
// InPollContext() so command submitters reached from a callback can tell
// they are nested and must fire-and-forget instead of waiting.
bool expected = false;
if (!g_pollActive.compare_exchange_strong(expected, true,
std::memory_order_acquire)) {
return; // another context is draining; it will reap our events
return PollResult::Busy; // another context is draining
}
g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed);
// Bound the work per call so a flooding/wedged device can never spin
// here forever; the outer wall-clock timeouts then fire instead of
// freezing. 4x the ring size is far above any legitimate burst.
constexpr uint32_t MAX_EVENTS_PER_CALL = EVT_RING_SIZE * 4;
// Bound each pass so a flooding/wedged device cannot monopolize its
// caller. Synchronous poll loops use a generous batch; the timer uses
// a much smaller recovery slice after the hard IRQ masks the source.
if (maxEvents == 0) maxEvents = 1;
uint32_t processed = 0, portEvts = 0, xferEvts = 0;
uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0;
while (processed < MAX_EVENTS_PER_CALL) {
while (processed < maxEvents) {
TRB& evt = g_evtRing[g_evtRingDequeue];
// Check if the cycle bit matches our expected cycle state
@@ -506,8 +540,8 @@ namespace Drivers::USB::Xhci {
// Defer enumeration to ProcessDeferredWork outside interrupt context.
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1] = true;
g_deferredWorkPending = true;
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
break;
}
@@ -616,9 +650,16 @@ namespace Drivers::USB::Xhci {
nullptr, 0, completionCode);
}
} else {
KernelLogStream(WARNING, "xHCI") << "Transfer error on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)epDci
<< " cc=" << (uint64_t)completionCode;
// Never print from this path: PollEvents may be
// running in the MSI handler. Defer a rate-limited
// diagnostic to idle/process context instead.
if (g_transferErrorReports.load(std::memory_order_relaxed) < 8) {
g_lastErrorSlot.store(slotId, std::memory_order_relaxed);
g_lastErrorEp.store(epDci, std::memory_order_relaxed);
g_lastErrorCc.store(completionCode, std::memory_order_relaxed);
g_transferErrorsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
@@ -651,37 +692,70 @@ namespace Drivers::USB::Xhci {
WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32));
// A full batch means the ring is being flooded -- surface the dominant
// event source (rate-limited) so a wedge is diagnosable, not silent.
if (processed >= MAX_EVENTS_PER_CALL) {
static uint32_t stormLogs = 0;
if (stormLogs < 8) {
stormLogs++;
KernelLogStream(WARNING, "xHCI") << "Event storm: " << (uint64_t)processed
<< "/call (port=" << (uint64_t)portEvts << " xfer=" << (uint64_t)xferEvts
<< " lastType=" << (uint64_t)lastType << " slot=" << (uint64_t)lastSlot
<< " ep=" << (uint64_t)lastEp << " cc=" << (uint64_t)lastCC << ")";
// A full batch means the ring is being flooded. Record the dominant
// source for a deferred, rate-limited process-context diagnostic.
bool exhausted = processed >= maxEvents;
if (exhausted) {
if (g_stormReports.load(std::memory_order_relaxed) < 8) {
g_lastStormProcessed.store(processed, std::memory_order_relaxed);
g_lastStormPortEvents.store(portEvts, std::memory_order_relaxed);
g_lastStormTransferEvents.store(xferEvts, std::memory_order_relaxed);
g_lastStormType.store(lastType, std::memory_order_relaxed);
g_lastStormSlot.store(lastSlot, std::memory_order_relaxed);
g_lastStormEp.store(lastEp, std::memory_order_relaxed);
g_lastStormCc.store(lastCC, std::memory_order_relaxed);
g_stormsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
}
g_pollOwnerCpu.store(-1, std::memory_order_relaxed);
g_pollActive.store(false, std::memory_order_release);
return exhausted ? PollResult::BudgetExhausted : PollResult::Drained;
}
void PollEvents() {
PollResult result = PollEventsBounded(EVT_RING_SIZE * 4);
if (result == PollResult::Drained &&
g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
// The hard IRQ leaves the interrupter masked. A synchronous
// process-context poll may drain the ring before an idle bottom
// half sees it, so finish the handoff here as well. Verify once
// more while masked: on another CPU, the IRQ may have acknowledged
// IMAN.IP for an event posted just after our first empty check. In
// that race, blindly enabling IE would strand the event because
// the IRQ already cleared its pending bit.
PollResult verify = PollEventsBounded(EVT_RING_SIZE * 4);
if (verify != PollResult::Drained) {
g_irqDrainPending.store(true, std::memory_order_release);
return;
}
// If hardware queues an event after this masked verification,
// IMAN.IP remains asserted and enabling IE generates a fresh MSI.
WriteRt(IR0_IMAN, IMAN_IE);
}
}
// -------------------------------------------------------------------------
// HandleInterrupt
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Clear USBSTS.EINT only (don't accidentally clear other W1C bits)
WriteOp(OP_USBSTS, USBSTS_EINT);
// Clear IMAN.IP and ensure IE stays enabled
WriteRt(IR0_IMAN, IMAN_IP | IMAN_IE);
PollEvents();
// Keep the hard IRQ strictly bounded. PollEvents dispatches HID,
// Bluetooth, and SDR callbacks; those callbacks can take subsystem
// locks, wake scheduler objects, re-arm endpoints, and copy sizeable
// buffers. Running them here lets a USB burst monopolize this
// higher-priority vector ahead of the LAPIC timer and can deadlock if
// an interrupted context owns one of those locks. Acknowledge and mask
// the source, then let an idle/process-context poll drain the ring.
WriteRt(IR0_IMAN, IMAN_IP);
g_irqDrainPending.store(true, std::memory_order_release);
}
// -------------------------------------------------------------------------
@@ -689,6 +763,8 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
uint32_t SendCommand(const TRB& trb) {
g_commandLock.Acquire();
// Place TRB at current enqueue position
TRB& slot = g_cmdRing[g_cmdRingEnqueue];
slot.Parameter0 = trb.Parameter0;
@@ -728,7 +804,9 @@ namespace Drivers::USB::Xhci {
while (Timekeeping::GetMilliseconds() - cmdStart < 2000) {
PollEvents();
if (g_cmdCompleted) {
return g_cmdCompletionCode;
uint32_t result = g_cmdCompletionCode;
g_commandLock.Release();
return result;
}
// Small delay
for (int j = 0; j < 100; j++) {
@@ -736,6 +814,9 @@ namespace Drivers::USB::Xhci {
}
}
// Do not hold the command-ring lock while taking the terminal mutex.
// A later caller may attempt recovery even if this command timed out.
g_commandLock.Release();
KernelLogStream(WARNING, "xHCI") << "Command timeout";
return 0xFF;
}
@@ -1304,7 +1385,55 @@ namespace Drivers::USB::Xhci {
}
bool HasDeferredWork() {
return g_initialized && g_deferredWorkPending;
return g_initialized &&
(g_irqDrainPending.load(std::memory_order_acquire) ||
g_deferredWorkPending.load(std::memory_order_acquire) ||
g_diagnosticWorkPending.load(std::memory_order_acquire));
}
void ServiceDeferredIrqEvents(uint32_t budget) {
if (!g_initialized ||
!g_irqDrainPending.load(std::memory_order_acquire)) return;
PollResult result = PollEventsBounded(budget);
if (result != PollResult::Drained) return;
// Clear before re-enabling. If hardware queued an event after the ring
// appeared empty, IMAN.IP remains asserted and enabling IE generates a
// fresh MSI; no event can be lost in this handoff.
if (g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
WriteRt(IR0_IMAN, IMAN_IE);
}
}
static void PrintDeferredDiagnostics() {
// Clear the wakeup before consuming counters. A producer racing after
// this store sets it again, so deferred work cannot be lost.
g_diagnosticWorkPending.store(false, std::memory_order_release);
uint32_t errors = g_transferErrorsPending.exchange(0, std::memory_order_acq_rel);
if (errors != 0 &&
g_transferErrorReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)errors
<< " transfer error(s); last slot="
<< (uint64_t)g_lastErrorSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastErrorEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastErrorCc.load(std::memory_order_relaxed);
}
uint32_t storms = g_stormsPending.exchange(0, std::memory_order_acq_rel);
if (storms != 0 && g_stormReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)storms
<< " event storm(s); last batch="
<< (uint64_t)g_lastStormProcessed.load(std::memory_order_relaxed)
<< " (port=" << (uint64_t)g_lastStormPortEvents.load(std::memory_order_relaxed)
<< " xfer=" << (uint64_t)g_lastStormTransferEvents.load(std::memory_order_relaxed)
<< " type=" << (uint64_t)g_lastStormType.load(std::memory_order_relaxed)
<< " slot=" << (uint64_t)g_lastStormSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastStormEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastStormCc.load(std::memory_order_relaxed) << ")";
}
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
@@ -1339,7 +1468,19 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
void ProcessDeferredWork() {
if (!g_initialized || !g_bootScanComplete || !g_deferredWorkPending) return;
if (!g_initialized) return;
// The hard IRQ only acknowledges and masks the interrupter. Drain a
// bounded slice here, outside interrupt context, so callbacks may use
// their normal locks without blocking the timer or scheduler. A busy
// process that is synchronously waiting for USB calls PollEvents()
// itself, which performs the same unmask handoff when the ring is dry.
ServiceDeferredIrqEvents(64);
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
bool haveHotplug = g_bootScanComplete &&
g_deferredWorkPending.load(std::memory_order_acquire);
if (!haveDiagnostics && !haveHotplug) return;
// CAS claim: any idle core may call this, but only one runs at a time.
// AcquireTransport in MassStorage handles the SCSI-in-flight case itself.
@@ -1348,11 +1489,17 @@ namespace Drivers::USB::Xhci {
std::memory_order_acquire, std::memory_order_relaxed)) {
return;
}
g_deferredWorkPending = false;
if (haveDiagnostics) PrintDeferredDiagnostics();
if (!haveHotplug) {
g_hotplugProcessing.store(false, std::memory_order_release);
return;
}
g_deferredWorkPending.store(false, std::memory_order_release);
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!g_hotplugPending[port]) continue;
g_hotplugPending[port] = false;
if (!g_hotplugPending[port].exchange(false, std::memory_order_acq_rel)) continue;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);
kernel/src/Drivers/USB/Xhci.hpp
+11 -4
View File
@@ -20,9 +20,11 @@ namespace Drivers::USB::Xhci {
constexpr uint32_t EVT_RING_SIZE = 64;
constexpr uint32_t XFER_RING_SIZE = 32;
// MSI configuration (E1000E uses IRQ 24/vector 56, we use 25/57)
constexpr uint8_t MSI_IRQ = 25;
constexpr uint32_t MSI_VECTOR = 57;
// MSI configuration. Keep xHCI distinct from AHCI's IRQ 25/vector 57;
// the IRQ dispatcher has one handler per slot, so sharing would replace
// one driver's handler and eventually strand USB or storage completions.
constexpr uint8_t MSI_IRQ = 29;
constexpr uint32_t MSI_VECTOR = 61;
constexpr uint32_t MSI_ADDR_BASE = 0xFEE00000;
// PCI class/subclass/progif for xHCI
@@ -299,6 +301,10 @@ namespace Drivers::USB::Xhci {
// Deferred hot-plug processing (call outside interrupt context)
void ProcessDeferredWork();
// Drain a bounded event batch after the hard IRQ masked the interrupter.
// Call only from process/idle context; this dispatches class callbacks.
void ServiceDeferredIrqEvents(uint32_t budget);
// Send a command on the command ring, wait for completion.
// Returns completion code.
uint32_t SendCommand(const TRB& trb);
@@ -367,7 +373,8 @@ namespace Drivers::USB::Xhci {
// Access device info
UsbDeviceInfo* GetDevice(uint8_t slotId);
// Poll event ring (called from interrupt handler or during init)
// Poll the event ring from process/idle context. The hard interrupt only
// acknowledges and masks the source, then queues deferred draining.
void PollEvents();
};