daniel/MontaukOS
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: kernel concurrency, interrupt context, and user VA safety

Browse Source
This commit is contained in:
daniel committed 2026-08-01 15:43:38 +02:00
1 parent f222bf1f04
commit b1c55073c7
71 files changed
+2654 -1092
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernel/src/Drivers/USB/Xhci.cpp
+200 -53
View File
@@ -54,9 +54,30 @@ namespace Drivers::USB::Xhci {
static bool g_bootScanComplete = false; // true after initial port scan finishes
// Hot-plug deferred work
static volatile bool g_hotplugPending[MAX_PORTS] = {};
static volatile bool g_deferredWorkPending = false;
static std::atomic<bool> g_hotplugPending[MAX_PORTS] = {};
static std::atomic<bool> g_deferredWorkPending{false};
static std::atomic<bool> g_diagnosticWorkPending{false};
static std::atomic<bool> g_hotplugProcessing{false};
static std::atomic<bool> g_irqDrainPending{false};
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
// a process-context mutex, so printing there can deadlock if the interrupt
// preempts a CPU that already owns the terminal lock. Capture the small
// diagnostic payload atomically and print it from ProcessDeferredWork().
static std::atomic<uint32_t> g_transferErrorsPending{0};
static std::atomic<uint32_t> g_transferErrorReports{0};
static std::atomic<uint32_t> g_lastErrorSlot{0};
static std::atomic<uint32_t> g_lastErrorEp{0};
static std::atomic<uint32_t> g_lastErrorCc{0};
static std::atomic<uint32_t> g_stormsPending{0};
static std::atomic<uint32_t> g_stormReports{0};
static std::atomic<uint32_t> g_lastStormProcessed{0};
static std::atomic<uint32_t> g_lastStormPortEvents{0};
static std::atomic<uint32_t> g_lastStormTransferEvents{0};
static std::atomic<uint32_t> g_lastStormType{0};
static std::atomic<uint32_t> g_lastStormSlot{0};
static std::atomic<uint32_t> g_lastStormEp{0};
static std::atomic<uint32_t> g_lastStormCc{0};
// MMIO region pointers
static volatile uint8_t* g_mmioBase = nullptr;
@@ -78,6 +99,12 @@ namespace Drivers::USB::Xhci {
static uint64_t g_cmdRingPhys = 0;
static uint32_t g_cmdRingEnqueue = 0;
static bool g_cmdRingCCS = true;
// The controller exposes one command ring and one command-completion
// mailbox. Endpoint recovery, hot-plug enumeration, and control-transfer
// recovery can originate on different CPUs, so the whole submit/wait
// transaction must be single-owner. This is a process/idle-context lock;
// SendCommand is never called from a transfer-event callback.
static kcp::Mutex g_commandLock;
// Event ring
static TRB* g_evtRing = nullptr;
@@ -125,12 +152,12 @@ namespace Drivers::USB::Xhci {
// completion. Used by InPollContext() / ControlTransfer().
//
// MUST be a real atomic claimed with compare-exchange, not a plain
// volatile bool: PollEvents runs from the MSI handler (BSP) AND from
// syscall-context pumps (A2DP WriteAudio/StartSource on whatever core the
// app runs on) AND from idle cores (ProcessDeferredWork). A check-then-set
// on a volatile is a cross-core TOCTOU: two cores both pass the check,
// both drain the shared event ring, g_evtRingDequeue/g_evtRingCCS desync,
// and completions get double-delivered (duplicate ACL packets) or lost
// volatile bool: PollEvents runs from syscall-context pumps (A2DP
// WriteAudio/StartSource on whatever core the app runs on) and from idle
// cores (ProcessDeferredWork). A check-then-set on a volatile is a
// cross-core TOCTOU: two cores both pass the check, both drain the shared
// event ring, g_evtRingDequeue/g_evtRingCCS desync, and completions get
// double-delivered (duplicate ACL packets) or lost
// (missed AVDTP responses) -- both observed on HW once audio streaming
// started pumping from syscall context.
static std::atomic<bool> g_pollActive{false};
@@ -293,7 +320,7 @@ namespace Drivers::USB::Xhci {
// Forward declarations
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq);
static void HandleInterrupt(uint8_t irq, bool fromUser);
// -------------------------------------------------------------------------
// BIOS-to-OS handoff (USB Legacy Support)
@@ -389,10 +416,13 @@ namespace Drivers::USB::Xhci {
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
if (cap == 0) {
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability not found";
return false;
}
Pci::DisableInterruptDelivery(bus, dev, func);
KernelLogStream(INFO, "xHCI") << "MSI capability at offset " << base::hex << (uint64_t)cap;
// Read Message Control (cap+2)
@@ -448,33 +478,37 @@ namespace Drivers::USB::Xhci {
// PollEvents - process event ring
// -------------------------------------------------------------------------
void PollEvents() {
// PollEvents runs both from the synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and from the xHCI MSI handler. On a
// single core the IRQ can preempt a poll loop mid-drain; if both advance
// g_evtRingDequeue / g_evtRingCCS the ring tracking desyncs and the
// cycle-bit check can start matching stale entries forever -> the boot
// freezes (observed wedging the Bluetooth firmware download at ~635 KB,
// where the dying device floods the event ring). Guard against re-entry:
// the interrupt is already acked (IMAN.IP cleared in HandleInterrupt) and
// the active poll loop drains these events itself. g_pollActive is also
// read by InPollContext() so command submitters (ControlTransfer) can
// tell they are nested and must fire-and-forget instead of waiting.
enum class PollResult : uint8_t {
Drained,
Busy,
BudgetExhausted,
};
static PollResult PollEventsBounded(uint32_t maxEvents) {
// PollEvents runs from synchronous poll loops (ControlTransfer,
// SendCommand, firmware download) and idle bottom halves on different
// CPUs. If two contexts advance g_evtRingDequeue / g_evtRingCCS, ring
// tracking desyncs and the cycle-bit check can start matching stale
// entries forever -> the boot freezes (observed wedging the Bluetooth
// firmware download at ~635 KB, where the dying device floods the event
// ring). Guard against cross-core re-entry. g_pollActive is also read by
// InPollContext() so command submitters reached from a callback can tell
// they are nested and must fire-and-forget instead of waiting.
bool expected = false;
if (!g_pollActive.compare_exchange_strong(expected, true,
std::memory_order_acquire)) {
return; // another context is draining; it will reap our events
return PollResult::Busy; // another context is draining
}
g_pollOwnerCpu.store(CurrentCpuIndex(), std::memory_order_relaxed);
// Bound the work per call so a flooding/wedged device can never spin
// here forever; the outer wall-clock timeouts then fire instead of
// freezing. 4x the ring size is far above any legitimate burst.
constexpr uint32_t MAX_EVENTS_PER_CALL = EVT_RING_SIZE * 4;
// Bound each pass so a flooding/wedged device cannot monopolize its
// caller. Synchronous poll loops use a generous batch; the timer uses
// a much smaller recovery slice after the hard IRQ masks the source.
if (maxEvents == 0) maxEvents = 1;
uint32_t processed = 0, portEvts = 0, xferEvts = 0;
uint32_t lastType = 0, lastSlot = 0, lastEp = 0, lastCC = 0;
while (processed < MAX_EVENTS_PER_CALL) {
while (processed < maxEvents) {
TRB& evt = g_evtRing[g_evtRingDequeue];
// Check if the cycle bit matches our expected cycle state
@@ -506,8 +540,8 @@ namespace Drivers::USB::Xhci {
// Defer enumeration to ProcessDeferredWork outside interrupt context.
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
g_hotplugPending[portId - 1] = true;
g_deferredWorkPending = true;
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
g_deferredWorkPending.store(true, std::memory_order_release);
}
break;
}
@@ -616,9 +650,16 @@ namespace Drivers::USB::Xhci {
nullptr, 0, completionCode);
}
} else {
KernelLogStream(WARNING, "xHCI") << "Transfer error on slot "
<< base::dec << (uint64_t)slotId << " ep " << (uint64_t)epDci
<< " cc=" << (uint64_t)completionCode;
// Never print from this path: PollEvents may be
// running in the MSI handler. Defer a rate-limited
// diagnostic to idle/process context instead.
if (g_transferErrorReports.load(std::memory_order_relaxed) < 8) {
g_lastErrorSlot.store(slotId, std::memory_order_relaxed);
g_lastErrorEp.store(epDci, std::memory_order_relaxed);
g_lastErrorCc.store(completionCode, std::memory_order_relaxed);
g_transferErrorsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
// Notify callback of errors too
if (g_transferCallbacks[slotId]) {
@@ -651,37 +692,70 @@ namespace Drivers::USB::Xhci {
WriteRt(IR0_ERDP, (uint32_t)(erdp & 0xFFFFFFFF));
WriteRt(IR0_ERDP + 4, (uint32_t)(erdp >> 32));
// A full batch means the ring is being flooded -- surface the dominant
// event source (rate-limited) so a wedge is diagnosable, not silent.
if (processed >= MAX_EVENTS_PER_CALL) {
static uint32_t stormLogs = 0;
if (stormLogs < 8) {
stormLogs++;
KernelLogStream(WARNING, "xHCI") << "Event storm: " << (uint64_t)processed
<< "/call (port=" << (uint64_t)portEvts << " xfer=" << (uint64_t)xferEvts
<< " lastType=" << (uint64_t)lastType << " slot=" << (uint64_t)lastSlot
<< " ep=" << (uint64_t)lastEp << " cc=" << (uint64_t)lastCC << ")";
// A full batch means the ring is being flooded. Record the dominant
// source for a deferred, rate-limited process-context diagnostic.
bool exhausted = processed >= maxEvents;
if (exhausted) {
if (g_stormReports.load(std::memory_order_relaxed) < 8) {
g_lastStormProcessed.store(processed, std::memory_order_relaxed);
g_lastStormPortEvents.store(portEvts, std::memory_order_relaxed);
g_lastStormTransferEvents.store(xferEvts, std::memory_order_relaxed);
g_lastStormType.store(lastType, std::memory_order_relaxed);
g_lastStormSlot.store(lastSlot, std::memory_order_relaxed);
g_lastStormEp.store(lastEp, std::memory_order_relaxed);
g_lastStormCc.store(lastCC, std::memory_order_relaxed);
g_stormsPending.fetch_add(1, std::memory_order_relaxed);
g_diagnosticWorkPending.store(true, std::memory_order_release);
}
}
g_pollOwnerCpu.store(-1, std::memory_order_relaxed);
g_pollActive.store(false, std::memory_order_release);
return exhausted ? PollResult::BudgetExhausted : PollResult::Drained;
}
void PollEvents() {
PollResult result = PollEventsBounded(EVT_RING_SIZE * 4);
if (result == PollResult::Drained &&
g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
// The hard IRQ leaves the interrupter masked. A synchronous
// process-context poll may drain the ring before an idle bottom
// half sees it, so finish the handoff here as well. Verify once
// more while masked: on another CPU, the IRQ may have acknowledged
// IMAN.IP for an event posted just after our first empty check. In
// that race, blindly enabling IE would strand the event because
// the IRQ already cleared its pending bit.
PollResult verify = PollEventsBounded(EVT_RING_SIZE * 4);
if (verify != PollResult::Drained) {
g_irqDrainPending.store(true, std::memory_order_release);
return;
}
// If hardware queues an event after this masked verification,
// IMAN.IP remains asserted and enabling IE generates a fresh MSI.
WriteRt(IR0_IMAN, IMAN_IE);
}
}
// -------------------------------------------------------------------------
// HandleInterrupt
// -------------------------------------------------------------------------
static void HandleInterrupt(uint8_t irq) {
static void HandleInterrupt(uint8_t irq, bool) {
(void)irq;
// Clear USBSTS.EINT only (don't accidentally clear other W1C bits)
WriteOp(OP_USBSTS, USBSTS_EINT);
// Clear IMAN.IP and ensure IE stays enabled
WriteRt(IR0_IMAN, IMAN_IP | IMAN_IE);
PollEvents();
// Keep the hard IRQ strictly bounded. PollEvents dispatches HID,
// Bluetooth, and SDR callbacks; those callbacks can take subsystem
// locks, wake scheduler objects, re-arm endpoints, and copy sizeable
// buffers. Running them here lets a USB burst monopolize this
// higher-priority vector ahead of the LAPIC timer and can deadlock if
// an interrupted context owns one of those locks. Acknowledge and mask
// the source, then let an idle/process-context poll drain the ring.
WriteRt(IR0_IMAN, IMAN_IP);
g_irqDrainPending.store(true, std::memory_order_release);
}
// -------------------------------------------------------------------------
@@ -689,6 +763,8 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
uint32_t SendCommand(const TRB& trb) {
g_commandLock.Acquire();
// Place TRB at current enqueue position
TRB& slot = g_cmdRing[g_cmdRingEnqueue];
slot.Parameter0 = trb.Parameter0;
@@ -728,7 +804,9 @@ namespace Drivers::USB::Xhci {
while (Timekeeping::GetMilliseconds() - cmdStart < 2000) {
PollEvents();
if (g_cmdCompleted) {
return g_cmdCompletionCode;
uint32_t result = g_cmdCompletionCode;
g_commandLock.Release();
return result;
}
// Small delay
for (int j = 0; j < 100; j++) {
@@ -736,6 +814,9 @@ namespace Drivers::USB::Xhci {
}
}
// Do not hold the command-ring lock while taking the terminal mutex.
// A later caller may attempt recovery even if this command timed out.
g_commandLock.Release();
KernelLogStream(WARNING, "xHCI") << "Command timeout";
return 0xFF;
}
@@ -1304,7 +1385,55 @@ namespace Drivers::USB::Xhci {
}
bool HasDeferredWork() {
return g_initialized && g_deferredWorkPending;
return g_initialized &&
(g_irqDrainPending.load(std::memory_order_acquire) ||
g_deferredWorkPending.load(std::memory_order_acquire) ||
g_diagnosticWorkPending.load(std::memory_order_acquire));
}
void ServiceDeferredIrqEvents(uint32_t budget) {
if (!g_initialized ||
!g_irqDrainPending.load(std::memory_order_acquire)) return;
PollResult result = PollEventsBounded(budget);
if (result != PollResult::Drained) return;
// Clear before re-enabling. If hardware queued an event after the ring
// appeared empty, IMAN.IP remains asserted and enabling IE generates a
// fresh MSI; no event can be lost in this handoff.
if (g_irqDrainPending.exchange(false, std::memory_order_acq_rel)) {
WriteRt(IR0_IMAN, IMAN_IE);
}
}
static void PrintDeferredDiagnostics() {
// Clear the wakeup before consuming counters. A producer racing after
// this store sets it again, so deferred work cannot be lost.
g_diagnosticWorkPending.store(false, std::memory_order_release);
uint32_t errors = g_transferErrorsPending.exchange(0, std::memory_order_acq_rel);
if (errors != 0 &&
g_transferErrorReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)errors
<< " transfer error(s); last slot="
<< (uint64_t)g_lastErrorSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastErrorEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastErrorCc.load(std::memory_order_relaxed);
}
uint32_t storms = g_stormsPending.exchange(0, std::memory_order_acq_rel);
if (storms != 0 && g_stormReports.fetch_add(1, std::memory_order_relaxed) < 8) {
KernelLogStream(WARNING, "xHCI") << base::dec << (uint64_t)storms
<< " event storm(s); last batch="
<< (uint64_t)g_lastStormProcessed.load(std::memory_order_relaxed)
<< " (port=" << (uint64_t)g_lastStormPortEvents.load(std::memory_order_relaxed)
<< " xfer=" << (uint64_t)g_lastStormTransferEvents.load(std::memory_order_relaxed)
<< " type=" << (uint64_t)g_lastStormType.load(std::memory_order_relaxed)
<< " slot=" << (uint64_t)g_lastStormSlot.load(std::memory_order_relaxed)
<< " ep=" << (uint64_t)g_lastStormEp.load(std::memory_order_relaxed)
<< " cc=" << (uint64_t)g_lastStormCc.load(std::memory_order_relaxed) << ")";
}
}
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
@@ -1339,7 +1468,19 @@ namespace Drivers::USB::Xhci {
// -------------------------------------------------------------------------
void ProcessDeferredWork() {
if (!g_initialized || !g_bootScanComplete || !g_deferredWorkPending) return;
if (!g_initialized) return;
// The hard IRQ only acknowledges and masks the interrupter. Drain a
// bounded slice here, outside interrupt context, so callbacks may use
// their normal locks without blocking the timer or scheduler. A busy
// process that is synchronously waiting for USB calls PollEvents()
// itself, which performs the same unmask handoff when the ring is dry.
ServiceDeferredIrqEvents(64);
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
bool haveHotplug = g_bootScanComplete &&
g_deferredWorkPending.load(std::memory_order_acquire);
if (!haveDiagnostics && !haveHotplug) return;
// CAS claim: any idle core may call this, but only one runs at a time.
// AcquireTransport in MassStorage handles the SCSI-in-flight case itself.
@@ -1348,11 +1489,17 @@ namespace Drivers::USB::Xhci {
std::memory_order_acquire, std::memory_order_relaxed)) {
return;
}
g_deferredWorkPending = false;
if (haveDiagnostics) PrintDeferredDiagnostics();
if (!haveHotplug) {
g_hotplugProcessing.store(false, std::memory_order_release);
return;
}
g_deferredWorkPending.store(false, std::memory_order_release);
for (uint32_t port = 0; port < g_maxPorts; port++) {
if (!g_hotplugPending[port]) continue;
g_hotplugPending[port] = false;
if (!g_hotplugPending[port].exchange(false, std::memory_order_acq_rel)) continue;
uint32_t portsc = ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE);