feat: aarch64 port - Raspberry Pi 4/400 SD boot (SDHCI, ext2 root + first-boot grow), USB hubs, PCI INTx routing, SMP, DMA fences
This commit is contained in:
daniel committed
1 parent
8c09ccef66
commit
808e1d66fd
66 files changed
+3690
-196
No files matched your search
@@ -191,7 +191,8 @@ namespace Drivers::Audio::IntelHda {
|
||||
// Write verb to CORB
|
||||
g_corb[wp] = verb;
|
||||
|
||||
// Update write pointer
|
||||
// Update write pointer once the verb is visible
|
||||
Arch::DmaWriteFence();
|
||||
Write16(REG_CORBWP, wp);
|
||||
|
||||
return true;
|
||||
@@ -202,6 +203,7 @@ namespace Drivers::Audio::IntelHda {
|
||||
uint16_t wp = Read16(REG_RIRBWP) & 0xFF;
|
||||
|
||||
if (g_rirbReadPtr != wp) {
|
||||
Arch::DmaReadFence();
|
||||
g_rirbReadPtr = (g_rirbReadPtr + 1) % g_rirbSize;
|
||||
|
||||
uint32_t ex = g_rirb[g_rirbReadPtr].ResponseEx;
|
||||
|
||||
@@ -14,11 +14,16 @@
|
||||
#include <Drivers/USB/Xhci.hpp>
|
||||
#include <Drivers/Storage/Ahci.hpp>
|
||||
#include <Drivers/Storage/Nvme.hpp>
|
||||
#include <Drivers/Storage/Sdhci.hpp>
|
||||
#include <Drivers/Storage/Gpt.hpp>
|
||||
#include <Drivers/Audio/IntelHda.hpp>
|
||||
#include <Graphics/Framebuffer.hpp>
|
||||
#include <Net/Net.hpp>
|
||||
#include <Terminal/Terminal.hpp>
|
||||
#include <ACPI/AcpiDevices.hpp>
|
||||
#include <Boot/Boot.hpp>
|
||||
#include <ACPI/AML/AmlInterpreter.hpp>
|
||||
#include <ACPI/AML/AmlResource.hpp>
|
||||
|
||||
namespace Drivers {
|
||||
|
||||
@@ -89,6 +94,10 @@ namespace Drivers {
|
||||
return Storage::Nvme::Probe(dev);
|
||||
}
|
||||
|
||||
static bool ProbeSdhci(const Pci::PciDevice& dev) {
|
||||
return Storage::Sdhci::Probe(dev);
|
||||
}
|
||||
|
||||
static bool ProbeIntelHda(const Pci::PciDevice& dev) {
|
||||
return Audio::IntelHda::Probe(dev);
|
||||
}
|
||||
@@ -189,6 +198,18 @@ namespace Drivers {
|
||||
// also reports 0x01, but it exposes an I/O-space BAR0:
|
||||
// ReadBar0 returns 0 for I/O BARs and Probe rejects it
|
||||
// cleanly, so matching it here is harmless.
|
||||
// SD host controller (standard interface) - class 0x08/0x05
|
||||
{
|
||||
"SDHCI",
|
||||
0,
|
||||
0x08, // ClassCode (Base system peripheral)
|
||||
0x05, // SubClass (SD host controller)
|
||||
0xFF,
|
||||
nullptr,
|
||||
0,
|
||||
Pci::ProbePhase::Normal,
|
||||
ProbeSdhci,
|
||||
},
|
||||
{
|
||||
"IntelHDA",
|
||||
0x8086, // VendorId (Intel)
|
||||
@@ -227,8 +248,117 @@ namespace Drivers {
|
||||
Graphics::DgpuPower::PowerDownUnused();
|
||||
}
|
||||
|
||||
// A device's _DMA (or its parent's) bounds the addresses it can reach.
|
||||
// DMA buffers come from anywhere in RAM, so RAM past the bound is a
|
||||
// configuration error: on the Raspberry Pi 4 the firmware's "Limit RAM
|
||||
// to 3 GB" setting must stay on.
|
||||
static void WarnIfRamBeyondDma(const char* devicePath) {
|
||||
char path[160];
|
||||
int len = 0;
|
||||
while (devicePath[len] != '\0' && len < 120) { path[len] = devicePath[len]; len++; }
|
||||
|
||||
Hal::AML::ResourceList dma;
|
||||
bool found = false;
|
||||
for (int attempt = 0; attempt < 2 && !found; attempt++) {
|
||||
const char suffix[] = "._DMA";
|
||||
for (int i = 0; i < (int)sizeof(suffix); i++) path[len + i] = suffix[i];
|
||||
Hal::AML::Object obj;
|
||||
if (Hal::AML::GetInterpreter().EvaluateObject(path, obj) &&
|
||||
obj.Type == Hal::AML::ObjectType::Buffer &&
|
||||
Hal::AML::ParseResourceTemplate(obj.Buffer.Data, obj.Buffer.Length, dma)) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
// Retry on the parent device.
|
||||
while (len > 0 && path[len - 1] != '.') len--;
|
||||
if (len == 0) return;
|
||||
len--;
|
||||
}
|
||||
|
||||
uint64_t dmaBase = 0, dmaLength = 0;
|
||||
if (!found || !dma.GetMemory(dmaBase, dmaLength) || dmaLength == 0) return;
|
||||
uint64_t dmaEnd = dmaBase + dmaLength;
|
||||
|
||||
const auto& map = montauk::boot::Info().memoryMap;
|
||||
for (uint64_t i = 0; i < map.count; i++) {
|
||||
const auto& r = map.regions[i];
|
||||
if (r.kind == montauk::boot::MemoryKind::Usable && r.base + r.length > dmaEnd) {
|
||||
Kt::KernelLogStream(Kt::WARNING, "Drivers") << devicePath << " can only reach RAM below 0x"
|
||||
<< base::hex << dmaEnd << ", but there is RAM up to 0x" << (r.base + r.length)
|
||||
<< ": its DMA may fail (Raspberry Pi 4: keep 'Limit RAM to 3 GB' enabled)";
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An xHCI controller ACPI describes as a platform device (PNP0D10)
|
||||
// instead of a PCI function: the Raspberry Pi 4/400's VL805 under UEFI
|
||||
// firmware in ACPI mode, where firmware has set up the non-ECAM PCIe
|
||||
// behind it. Its DMA is non-coherent unless _CCA says otherwise.
|
||||
static void ProbePlatformXhci() {
|
||||
if (USB::Xhci::IsInitialized()) return;
|
||||
|
||||
Hal::AcpiDevices::DeviceInfo info{};
|
||||
if (!Hal::AcpiDevices::FindDeviceByHid("PNP0D10", info) &&
|
||||
!Hal::AcpiDevices::FindDeviceByHid("PNP0D15", info)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Hal::AML::ResourceList resources;
|
||||
uint64_t base = 0, length = 0;
|
||||
if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
|
||||
!resources.GetMemory(base, length) || base == 0) {
|
||||
Kt::KernelLogStream(Kt::WARNING, "Drivers") << "xHCI " << info.Path
|
||||
<< ": no register block in _CRS";
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t gsi = 0;
|
||||
bool level = true, activeLow = false;
|
||||
if (auto* irq = resources.FindFirst(Hal::AML::ResourceType::ExtendedIrq)) {
|
||||
gsi = irq->ExtendedIrq.Interrupt;
|
||||
level = (irq->ExtendedIrq.Flags & 1) == 0;
|
||||
activeLow = (irq->ExtendedIrq.Flags & 2) != 0;
|
||||
}
|
||||
|
||||
bool coherent = false;
|
||||
Hal::AML::Object cca;
|
||||
if (Hal::AML::GetInterpreter().EvaluateChild(info.NodeIndex, "_CCA", nullptr, 0, cca) &&
|
||||
cca.Type == Hal::AML::ObjectType::Integer) {
|
||||
coherent = cca.Integer != 0;
|
||||
}
|
||||
|
||||
Kt::KernelLogStream(Kt::INFO, "Drivers") << "xHCI platform device " << info.Path
|
||||
<< " at 0x" << base::hex << base << ", GSI " << base::dec << (uint64_t)gsi
|
||||
<< (coherent ? ", coherent DMA" : ", non-coherent DMA");
|
||||
WarnIfRamBeyondDma(info.Path);
|
||||
USB::Xhci::ProbePlatform(base, length, gsi, level, activeLow, coherent);
|
||||
}
|
||||
|
||||
// SD host controllers ACPI describes directly: the Raspberry Pi 4's
|
||||
// EMMC2 (BRCME88C, the SD card slot) or a standard PNP0D40.
|
||||
static void ProbePlatformSdhci() {
|
||||
if (Storage::Sdhci::IsInitialized()) return;
|
||||
|
||||
Hal::AcpiDevices::DeviceInfo info{};
|
||||
if (!Hal::AcpiDevices::FindDeviceByHid("BRCME88C", info) &&
|
||||
!Hal::AcpiDevices::FindDeviceByHid("PNP0D40", info)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Hal::AML::ResourceList resources;
|
||||
uint64_t base = 0, length = 0;
|
||||
if (!Hal::AcpiDevices::EvaluateCrs(info.NodeIndex, resources) ||
|
||||
!resources.GetMemory(base, length) || base == 0) {
|
||||
return;
|
||||
}
|
||||
Storage::Sdhci::ProbePlatform(base, info.Path);
|
||||
}
|
||||
|
||||
void ProbeNormal() {
|
||||
Pci::ProbeAll(g_driverTable, g_driverTableCount, Pci::ProbePhase::Normal);
|
||||
ProbePlatformXhci();
|
||||
ProbePlatformSdhci();
|
||||
}
|
||||
|
||||
void InitializeNetwork() {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <Libraries/Memory.hpp>
|
||||
#include <Arch/Interrupts.hpp>
|
||||
#include <CppLib/Spinlock.hpp>
|
||||
#include <Arch/Cpu.hpp>
|
||||
#include <atomic>
|
||||
|
||||
using namespace Kt;
|
||||
@@ -491,6 +492,7 @@ namespace Drivers::Net::E1000 {
|
||||
|
||||
// Advance the tail pointer (tells the NIC there's a new packet)
|
||||
g_txTail = (g_txTail + 1) % TX_DESC_COUNT;
|
||||
Arch::DmaWriteFence();
|
||||
WriteReg(REG_TDT, g_txTail);
|
||||
|
||||
g_txPacketCount++;
|
||||
@@ -545,6 +547,7 @@ namespace Drivers::Net::E1000 {
|
||||
uint32_t nextIdx = (g_rxTail + 1) % RX_DESC_COUNT;
|
||||
RxDescriptor& desc = g_rxDescs[nextIdx];
|
||||
if (!(desc.Status & RXSTA_DD)) break;
|
||||
Arch::DmaReadFence();
|
||||
|
||||
uint16_t length = desc.Length;
|
||||
g_rxPacketCount++;
|
||||
@@ -554,6 +557,7 @@ namespace Drivers::Net::E1000 {
|
||||
desc.Length = 0;
|
||||
desc.Errors = 0;
|
||||
g_rxTail = nextIdx;
|
||||
Arch::DmaWriteFence();
|
||||
WriteReg(REG_RDT, g_rxTail);
|
||||
processed++;
|
||||
}
|
||||
|
||||
@@ -667,6 +667,7 @@ namespace Drivers::Net::E1000E {
|
||||
if (!(desc.Status & RXSTA_DD)) {
|
||||
break;
|
||||
}
|
||||
Arch::DmaReadFence();
|
||||
|
||||
uint16_t length = desc.Length;
|
||||
g_rxPacketCount++;
|
||||
@@ -680,6 +681,7 @@ namespace Drivers::Net::E1000E {
|
||||
desc.Errors = 0;
|
||||
|
||||
g_rxTail = nextIdx;
|
||||
Arch::DmaWriteFence();
|
||||
WriteReg(REG_RDT, g_rxTail);
|
||||
processed++;
|
||||
}
|
||||
@@ -709,6 +711,7 @@ namespace Drivers::Net::E1000E {
|
||||
desc.Status = 0;
|
||||
|
||||
g_txTail = (g_txTail + 1) % TX_DESC_COUNT;
|
||||
Arch::DmaWriteFence();
|
||||
WriteReg(REG_TDT, g_txTail);
|
||||
|
||||
g_txPacketCount++;
|
||||
|
||||
@@ -267,13 +267,15 @@ namespace Drivers::Storage::Ahci {
|
||||
}
|
||||
}
|
||||
|
||||
// Issue the command
|
||||
// Issue the command once its header, table and PRDs are visible
|
||||
Arch::DmaWriteFence();
|
||||
WritePortReg(port, PORT_CI, 1u << slot);
|
||||
|
||||
// Wait for completion
|
||||
for (int i = 0; i < 5000000; i++) {
|
||||
uint32_t ci = ReadPortReg(port, PORT_CI);
|
||||
if (!(ci & (1u << slot))) {
|
||||
Arch::DmaReadFence();
|
||||
// Check for errors
|
||||
uint32_t is = ReadPortReg(port, PORT_IS);
|
||||
if (is & PORT_IS_TFES) {
|
||||
@@ -641,10 +643,19 @@ namespace Drivers::Storage::Ahci {
|
||||
for (int i = 0; i < MAX_PORTS; i++) {
|
||||
if (!(g_portsImplemented & (1u << i))) continue;
|
||||
|
||||
// The signature arrives in a D2H FIS, which an HBA may only
|
||||
// latch into PxSIG once FIS receive is running (QEMU does so):
|
||||
// start every port with a device on its link, then classify.
|
||||
if ((ReadPortReg(i, PORT_SSTS) & SSTS_DET_MASK) != SSTS_DET_PRESENT) continue;
|
||||
InitPort(i);
|
||||
|
||||
PortType type = ClassifyPort(i);
|
||||
g_ports[i].Type = type;
|
||||
|
||||
if (type == PortType::None) continue;
|
||||
if (type == PortType::None) {
|
||||
StopPort(i);
|
||||
continue;
|
||||
}
|
||||
|
||||
const char* typeStr = "Unknown";
|
||||
switch (type) {
|
||||
@@ -658,8 +669,6 @@ namespace Drivers::Storage::Ahci {
|
||||
KernelLogStream(INFO, "AHCI") << "Port " << base::dec << (uint64_t)i
|
||||
<< ": " << typeStr << " device detected";
|
||||
|
||||
InitPort(i);
|
||||
|
||||
if (type == PortType::Sata) {
|
||||
if (IdentifyDevice(i)) {
|
||||
g_ports[i].Active = true;
|
||||
|
||||
@@ -17,6 +17,7 @@ namespace Drivers::Storage {
|
||||
BLOCK_KIND_SATAPI = 2,
|
||||
BLOCK_KIND_NVME = 3,
|
||||
BLOCK_KIND_USB_MSC = 4,
|
||||
BLOCK_KIND_SD = 5,
|
||||
};
|
||||
|
||||
struct BlockDevice {
|
||||
|
||||
@@ -107,6 +107,8 @@ namespace Drivers::Storage::Gpt {
|
||||
return GUID_LINUX_SWAP;
|
||||
case 0x83:
|
||||
return GUID_LINUX_FS;
|
||||
case 0xEF:
|
||||
return GUID_EFI_SYSTEM;
|
||||
default:
|
||||
return GUID_BASIC_DATA;
|
||||
}
|
||||
@@ -551,6 +553,78 @@ namespace Drivers::Storage::Gpt {
|
||||
return removed;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// MBR partition resize
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Read sector 0 into mbr and return the primary slot that holds `part`, or
|
||||
// -1 when the device is GPT (protective MBR) or `part` is not a primary
|
||||
// entry (a logical partition inside an extended one).
|
||||
static int FindMbrSlot(const PartitionInfo& part, ProtectiveMbr& mbr) {
|
||||
const BlockDevice* dev = GetBlockDevice(part.BlockDevIndex);
|
||||
if (!dev || !dev->ReadSectors(dev->Ctx, 0, 1, &mbr)) return -1;
|
||||
if (mbr.Signature != 0xAA55 || ValidateProtectiveMbr((const uint8_t*)&mbr)) return -1;
|
||||
|
||||
for (int i = 0; i < 4; i++) {
|
||||
const MbrPartitionEntry& e = mbr.Partitions[i];
|
||||
if (e.Type == 0 || IsExtendedMbrType(e.Type)) continue;
|
||||
if (e.LbaFirst == part.StartLba && e.SectorCount == part.SectorCount) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint64_t GetMbrPartitionMaxSectors(int index) {
|
||||
const PartitionInfo* part = GetPartition(index);
|
||||
if (!part) return 0;
|
||||
|
||||
ProtectiveMbr mbr;
|
||||
if (FindMbrSlot(*part, mbr) < 0) return 0;
|
||||
const BlockDevice* dev = GetBlockDevice(part->BlockDevIndex);
|
||||
|
||||
// Stop at whichever primary entry starts next (an extended entry covers
|
||||
// every logical partition inside it), else at the end of the device.
|
||||
uint64_t end = dev->SectorCount;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
const MbrPartitionEntry& e = mbr.Partitions[i];
|
||||
if (e.Type == 0 || e.SectorCount == 0) continue;
|
||||
if (e.LbaFirst > part->StartLba && e.LbaFirst < end) end = e.LbaFirst;
|
||||
}
|
||||
|
||||
// The entry stores a 32-bit sector count; keep the end addressable too.
|
||||
if (end > 0xFFFFFFFFull) end = 0xFFFFFFFFull;
|
||||
return end > part->StartLba ? end - part->StartLba : 0;
|
||||
}
|
||||
|
||||
int ResizeMbrPartition(int index, uint64_t sectorCount) {
|
||||
if (index < 0 || index >= g_partitionCount) return -1;
|
||||
PartitionInfo& part = g_partitions[index];
|
||||
if (sectorCount < part.SectorCount) return -1;
|
||||
if (sectorCount > GetMbrPartitionMaxSectors(index)) return -1;
|
||||
if (sectorCount == part.SectorCount) return 0;
|
||||
|
||||
ProtectiveMbr mbr;
|
||||
int slot = FindMbrSlot(part, mbr);
|
||||
if (slot < 0) return -1;
|
||||
|
||||
MbrPartitionEntry& e = mbr.Partitions[slot];
|
||||
e.SectorCount = (uint32_t)sectorCount;
|
||||
// CHS cannot address past 8 GiB; FE FF FF tells CHS-era tools to use
|
||||
// the LBA fields instead.
|
||||
e.ChsLast[0] = 0xFE;
|
||||
e.ChsLast[1] = 0xFF;
|
||||
e.ChsLast[2] = 0xFF;
|
||||
|
||||
const BlockDevice* dev = GetBlockDevice(part.BlockDevIndex);
|
||||
if (!dev->WriteSectors(dev->Ctx, 0, 1, &mbr)) return -1;
|
||||
if (dev->Flush) dev->Flush(dev->Ctx);
|
||||
|
||||
KernelLogStream(OK, "MBR") << "Partition " << index << " grown from "
|
||||
<< part.SectorCount << " to " << sectorCount << " sectors";
|
||||
part.SectorCount = sectorCount;
|
||||
part.EndLba = part.StartLba + sectorCount - 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// ASCII to UTF-16LE for partition names
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
@@ -127,6 +127,18 @@ namespace Drivers::Storage::Gpt {
|
||||
// hot-pluggable block device disappears.
|
||||
int RemovePartitionsForBlockDevice(int blockDevIndex);
|
||||
|
||||
// Largest size, in sectors, that partition `index` could be grown to: up to
|
||||
// the next partition on its device or the end of the device, within the
|
||||
// 32-bit LBA fields of an MBR entry. Returns 0 if the partition is not a
|
||||
// primary entry of a plain MBR disk (GPT disks and logical partitions are
|
||||
// not resized).
|
||||
uint64_t GetMbrPartitionMaxSectors(int index);
|
||||
|
||||
// Grow MBR partition `index` in place to sectorCount sectors, rewriting its
|
||||
// table entry and the in-memory record. Data is untouched. Returns 0 on
|
||||
// success, -1 if the partition cannot be resized to that size.
|
||||
int ResizeMbrPartition(int index, uint64_t sectorCount);
|
||||
|
||||
// Look up the human-readable name for a partition type GUID.
|
||||
const char* GetTypeName(const Guid& typeGuid);
|
||||
|
||||
|
||||
@@ -87,7 +87,9 @@ namespace Drivers::Storage::Nvme {
|
||||
// CQ y Head Doorbell offset = 0x1000 + ((2y+1) * doorbellStride)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// The submission queue entry must be visible before the doorbell.
|
||||
static void WriteSqTailDoorbell(uint16_t queueId, uint16_t value) {
|
||||
Arch::DmaWriteFence();
|
||||
uint32_t offset = 0x1000 + (2 * queueId) * g_doorbellStride;
|
||||
*(volatile uint32_t*)(g_mmioBase + offset) = value;
|
||||
}
|
||||
@@ -152,6 +154,8 @@ namespace Drivers::Storage::Nvme {
|
||||
uint16_t status = cqe->Status;
|
||||
|
||||
if ((status & CQE_PHASE_BIT) == phase) {
|
||||
// The rest of the entry (and the data) before the phase bit.
|
||||
Arch::DmaReadFence();
|
||||
uint16_t consumedId = cqe->CommandId;
|
||||
uint32_t result = cqe->Result;
|
||||
uint16_t sqHead = cqe->SqHead;
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
/*
|
||||
* Sdhci.cpp
|
||||
* SD Host Controller (SDHCI) driver: SD cards as block devices
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#include "Sdhci.hpp"
|
||||
#include "BlockDevice.hpp"
|
||||
#include <Arch/Cpu.hpp>
|
||||
#include <CppLib/Spinlock.hpp>
|
||||
#include <CppLib/Stream.hpp>
|
||||
#include <Libraries/Memory.hpp>
|
||||
#include <Memory/HHDM.hpp>
|
||||
#include <Memory/Paging.hpp>
|
||||
#include <Terminal/Terminal.hpp>
|
||||
#include <Timekeeping/Timer.hpp>
|
||||
|
||||
// One slot, polled, programmed I/O through the buffer data port. PIO keeps
|
||||
// the driver free of DMA address translation and cache coherency, which on
|
||||
// the Raspberry Pi 4 depend on the SoC stepping (EMMC2's _DMA).
|
||||
//
|
||||
// Every register access is 32 bits wide: the Broadcom controllers (the Pi's
|
||||
// EMMC2, and the Arasan-derived EMMC before it) do not support narrower
|
||||
// ones. 8/16-bit fields are written by read-modify-write of their dword,
|
||||
// and the transfer-mode/command pair in a single write that issues the
|
||||
// command.
|
||||
|
||||
using namespace Kt;
|
||||
|
||||
namespace Drivers::Storage::Sdhci {
|
||||
|
||||
// Registers (SD Host Controller Simplified Specification 3.00)
|
||||
static constexpr uint32_t REG_BLOCK_SIZE_COUNT = 0x04; // size [15:0], count [31:16]
|
||||
static constexpr uint32_t REG_ARGUMENT = 0x08;
|
||||
static constexpr uint32_t REG_XFER_MODE_CMD = 0x0C; // transfer mode [15:0], command [31:16]
|
||||
static constexpr uint32_t REG_RESPONSE = 0x10; // 4 dwords
|
||||
static constexpr uint32_t REG_BUFFER_DATA = 0x20;
|
||||
static constexpr uint32_t REG_PRESENT_STATE = 0x24;
|
||||
static constexpr uint32_t REG_HOST_CONTROL = 0x28; // host ctl 1 [7:0], power [15:8]
|
||||
static constexpr uint32_t REG_CLOCK_CONTROL = 0x2C; // clock [15:0], timeout [23:16], reset [31:24]
|
||||
static constexpr uint32_t REG_INT_STATUS = 0x30; // normal [15:0], error [31:16]
|
||||
static constexpr uint32_t REG_INT_ENABLE = 0x34; // status enables
|
||||
static constexpr uint32_t REG_INT_SIGNAL = 0x38; // interrupt signal enables
|
||||
static constexpr uint32_t REG_CAPABILITIES = 0x40;
|
||||
static constexpr uint32_t REG_VERSION = 0xFC; // host version [31:16]
|
||||
|
||||
// Present state
|
||||
static constexpr uint32_t PS_CMD_INHIBIT = 1u << 0;
|
||||
static constexpr uint32_t PS_DAT_INHIBIT = 1u << 1;
|
||||
|
||||
// Interrupt status
|
||||
static constexpr uint32_t INT_CMD_COMPLETE = 1u << 0;
|
||||
static constexpr uint32_t INT_XFER_COMPLETE = 1u << 1;
|
||||
static constexpr uint32_t INT_WRITE_READY = 1u << 4;
|
||||
static constexpr uint32_t INT_READ_READY = 1u << 5;
|
||||
static constexpr uint32_t INT_ERROR = 1u << 15;
|
||||
static constexpr uint32_t INT_ERROR_MASK = 0xFFFF0000u;
|
||||
|
||||
// Host control 1 / power control
|
||||
static constexpr uint32_t HC_4BIT = 1u << 1;
|
||||
static constexpr uint32_t PWR_ON = 1u << 8;
|
||||
static constexpr uint32_t PWR_330 = 7u << 9;
|
||||
static constexpr uint32_t PWR_300 = 6u << 9;
|
||||
static constexpr uint32_t PWR_180 = 5u << 9;
|
||||
|
||||
// Clock control
|
||||
static constexpr uint32_t CLK_INT_EN = 1u << 0;
|
||||
static constexpr uint32_t CLK_INT_STABLE = 1u << 1;
|
||||
static constexpr uint32_t CLK_SD_EN = 1u << 2;
|
||||
static constexpr uint32_t TIMEOUT_MAX = 0xEu << 16;
|
||||
|
||||
// Software reset (byte 0x2F)
|
||||
static constexpr uint32_t RESET_ALL = 1u << 24;
|
||||
static constexpr uint32_t RESET_CMD = 1u << 25;
|
||||
static constexpr uint32_t RESET_DAT = 1u << 26;
|
||||
|
||||
// Capabilities
|
||||
static constexpr uint32_t CAP_VOLT_330 = 1u << 24;
|
||||
static constexpr uint32_t CAP_VOLT_300 = 1u << 25;
|
||||
|
||||
// Transfer mode
|
||||
static constexpr uint32_t XM_BLOCK_COUNT_EN = 1u << 1;
|
||||
static constexpr uint32_t XM_AUTO_CMD12 = 1u << 2;
|
||||
static constexpr uint32_t XM_READ = 1u << 4;
|
||||
static constexpr uint32_t XM_MULTI_BLOCK = 1u << 5;
|
||||
|
||||
// Command register flags
|
||||
enum Response : uint32_t {
|
||||
RespNone = 0,
|
||||
Resp136 = 1, // R2
|
||||
Resp48 = 2 | 3u << 3, // R1, R6, R7: CRC and index checked
|
||||
Resp48Busy= 3 | 3u << 3, // R1b
|
||||
Resp48NoCrc = 2, // R3
|
||||
};
|
||||
static constexpr uint32_t CMD_DATA = 1u << 5;
|
||||
|
||||
static constexpr uint32_t BlockSize = 512;
|
||||
// The block count register is 16 bits wide.
|
||||
static constexpr uint32_t MaxBlocksPerCommand = 0xFFFF;
|
||||
static constexpr uint64_t CommandTimeoutMs = 200;
|
||||
static constexpr uint64_t DataTimeoutMs = 1000;
|
||||
|
||||
// Base clock when the capabilities leave it to the platform: the
|
||||
// Raspberry Pi 4's EMMC2 runs from a 100 MHz clock the firmware sets up.
|
||||
static constexpr uint32_t FallbackBaseClockHz = 100000000;
|
||||
|
||||
static volatile uint8_t* g_regs = nullptr;
|
||||
static bool g_initialized = false;
|
||||
static uint32_t g_version = 0; // 0: 1.00, 1: 2.00, 2: 3.00
|
||||
static uint32_t g_baseClockHz = 0;
|
||||
static uint32_t g_rca = 0;
|
||||
static bool g_highCapacity = false;
|
||||
static uint64_t g_sectorCount = 0;
|
||||
static kcp::Mutex g_lock;
|
||||
|
||||
static inline uint32_t Read(uint32_t reg) {
|
||||
return *(volatile uint32_t*)(g_regs + reg);
|
||||
}
|
||||
|
||||
static inline void Write(uint32_t reg, uint32_t value) {
|
||||
*(volatile uint32_t*)(g_regs + reg) = value;
|
||||
}
|
||||
|
||||
static bool WaitClear(uint32_t reg, uint32_t mask, uint64_t timeoutMs) {
|
||||
uint64_t start = Timekeeping::GetMilliseconds();
|
||||
while (Read(reg) & mask) {
|
||||
if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
|
||||
Arch::Pause();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool Reset(uint32_t which) {
|
||||
Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | which);
|
||||
return WaitClear(REG_CLOCK_CONTROL, which, 100);
|
||||
}
|
||||
|
||||
// Wait for any of `bits` in the normal status, or an error. Clears what
|
||||
// it returns on.
|
||||
static bool WaitStatus(uint32_t bits, uint64_t timeoutMs) {
|
||||
uint64_t start = Timekeeping::GetMilliseconds();
|
||||
for (;;) {
|
||||
uint32_t status = Read(REG_INT_STATUS);
|
||||
if (status & (INT_ERROR | INT_ERROR_MASK)) {
|
||||
Write(REG_INT_STATUS, status);
|
||||
Reset(RESET_CMD | RESET_DAT);
|
||||
return false;
|
||||
}
|
||||
if (status & bits) {
|
||||
Write(REG_INT_STATUS, status & bits);
|
||||
return true;
|
||||
}
|
||||
if (Timekeeping::GetMilliseconds() - start > timeoutMs) {
|
||||
Reset(RESET_CMD | RESET_DAT);
|
||||
return false;
|
||||
}
|
||||
Arch::Pause();
|
||||
}
|
||||
}
|
||||
|
||||
static bool SendCommand(uint32_t index, uint32_t argument, uint32_t response,
|
||||
uint32_t transferMode = 0, bool data = false) {
|
||||
uint32_t inhibit = PS_CMD_INHIBIT;
|
||||
if (data || response == Resp48Busy) inhibit |= PS_DAT_INHIBIT;
|
||||
if (!WaitClear(REG_PRESENT_STATE, inhibit, CommandTimeoutMs)) {
|
||||
Reset(RESET_CMD | RESET_DAT);
|
||||
return false;
|
||||
}
|
||||
|
||||
Write(REG_INT_STATUS, 0xFFFFFFFF);
|
||||
Write(REG_ARGUMENT, argument);
|
||||
uint32_t command = index << 8 | response | (data ? CMD_DATA : 0);
|
||||
Write(REG_XFER_MODE_CMD, command << 16 | transferMode);
|
||||
|
||||
if (!WaitStatus(INT_CMD_COMPLETE, CommandTimeoutMs)) return false;
|
||||
if (response == Resp48Busy && !WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// An application command: CMD55 with the card's address first.
|
||||
static bool SendAppCommand(uint32_t index, uint32_t argument, uint32_t response) {
|
||||
return SendCommand(55, g_rca << 16, Resp48) && SendCommand(index, argument, response);
|
||||
}
|
||||
|
||||
// Bits [start, start+length) of a 136-bit response (CID/CSD numbering).
|
||||
// The controller drops the CRC byte, so CSD bit n is response bit n-8.
|
||||
static uint32_t ResponseBits(const uint32_t response[4], int start, int length) {
|
||||
uint32_t value = 0;
|
||||
for (int i = 0; i < length; i++) {
|
||||
int bit = start + i - 8;
|
||||
if (response[bit / 32] & (1u << (bit % 32))) value |= 1u << i;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
static bool SetClock(uint32_t hz) {
|
||||
uint32_t clock = Read(REG_CLOCK_CONTROL);
|
||||
Write(REG_CLOCK_CONTROL, clock & ~(0xFFFFu));
|
||||
|
||||
uint32_t divider = 0; // base clock / (2 * divider); 0 = undivided
|
||||
if (hz < g_baseClockHz) {
|
||||
divider = (g_baseClockHz + 2 * hz - 1) / (2 * hz);
|
||||
uint32_t max = g_version >= 2 ? 0x3FF : 0x80;
|
||||
if (divider > max) divider = max;
|
||||
if (g_version < 2) {
|
||||
// Before 3.00 the divider is a power of two.
|
||||
uint32_t p = 1;
|
||||
while (p < divider) p <<= 1;
|
||||
divider = p;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t field = (divider & 0xFF) << 8 | ((divider >> 8) & 0x3) << 6;
|
||||
clock = (Read(REG_CLOCK_CONTROL) & ~0xFFFFu) | field | CLK_INT_EN;
|
||||
Write(REG_CLOCK_CONTROL, clock);
|
||||
|
||||
uint64_t start = Timekeeping::GetMilliseconds();
|
||||
while (!(Read(REG_CLOCK_CONTROL) & CLK_INT_STABLE)) {
|
||||
if (Timekeeping::GetMilliseconds() - start > 20) return false;
|
||||
Arch::Pause();
|
||||
}
|
||||
Write(REG_CLOCK_CONTROL, Read(REG_CLOCK_CONTROL) | CLK_SD_EN);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool InitializeCard() {
|
||||
uint32_t response[4];
|
||||
|
||||
if (!SendCommand(0, 0, RespNone)) return false;
|
||||
|
||||
// CMD8: a 2.00+ card echoes the check pattern.
|
||||
bool v2 = SendCommand(8, 0x1AA, Resp48) && (Read(REG_RESPONSE) & 0xFFF) == 0x1AA;
|
||||
|
||||
// ACMD41 until the card leaves its busy state (up to a second).
|
||||
uint32_t ocr = 0;
|
||||
uint64_t start = Timekeeping::GetMilliseconds();
|
||||
for (;;) {
|
||||
if (!SendAppCommand(41, (v2 ? 1u << 30 : 0) | 0x00FF8000, Resp48NoCrc)) return false;
|
||||
ocr = Read(REG_RESPONSE);
|
||||
if (ocr & (1u << 31)) break;
|
||||
if (Timekeeping::GetMilliseconds() - start > 1000) return false;
|
||||
}
|
||||
g_highCapacity = (ocr & (1u << 30)) != 0;
|
||||
|
||||
if (!SendCommand(2, 0, Resp136)) return false;
|
||||
if (!SendCommand(3, 0, Resp48)) return false;
|
||||
g_rca = Read(REG_RESPONSE) >> 16;
|
||||
|
||||
if (!SendCommand(9, g_rca << 16, Resp136)) return false;
|
||||
for (int i = 0; i < 4; i++) response[i] = Read(REG_RESPONSE + i * 4);
|
||||
if (ResponseBits(response, 126, 2) == 1) {
|
||||
uint64_t cSize = ResponseBits(response, 48, 22);
|
||||
g_sectorCount = (cSize + 1) * 1024;
|
||||
} else {
|
||||
uint64_t cSize = ResponseBits(response, 62, 12);
|
||||
uint64_t mult = ResponseBits(response, 47, 3);
|
||||
uint64_t readBlockLength = ResponseBits(response, 80, 4);
|
||||
g_sectorCount = ((cSize + 1) << (mult + 2)) * (1ULL << readBlockLength) / BlockSize;
|
||||
}
|
||||
|
||||
if (!SendCommand(7, g_rca << 16, Resp48Busy)) return false;
|
||||
if (!g_highCapacity && !SendCommand(16, BlockSize, Resp48)) return false;
|
||||
|
||||
// 4-bit bus when the card agrees; 1-bit otherwise.
|
||||
if (SendAppCommand(6, 2, Resp48)) {
|
||||
Write(REG_HOST_CONTROL, Read(REG_HOST_CONTROL) | HC_4BIT);
|
||||
}
|
||||
|
||||
SetClock(25000000);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Move one block through the buffer data port. Caller buffers are only
|
||||
// byte aligned in general, so unaligned ones go through a bounce copy.
|
||||
static void PioBlock(uint8_t* data, bool write) {
|
||||
uint32_t bounce[BlockSize / 4];
|
||||
bool aligned = ((uintptr_t)data & 3) == 0;
|
||||
uint32_t* words = aligned ? (uint32_t*)data : bounce;
|
||||
if (write && !aligned) memcpy(bounce, data, BlockSize);
|
||||
for (uint32_t i = 0; i < BlockSize / 4; i++) {
|
||||
if (write) Write(REG_BUFFER_DATA, words[i]);
|
||||
else words[i] = Read(REG_BUFFER_DATA);
|
||||
}
|
||||
if (!write && !aligned) memcpy(data, bounce, BlockSize);
|
||||
}
|
||||
|
||||
// One command for `count` consecutive blocks: CMD17/CMD24 for a single
|
||||
// block, else CMD18/CMD25 with the controller sending the closing CMD12
|
||||
// itself (Auto CMD12; the BCM2711's EMMC2 wants it for multi-block reads).
|
||||
// Cards program a multi-block write a flash page at a time; issued one
|
||||
// block per command, each 512 bytes costs a full page program, which made
|
||||
// writes (the first-boot grow above all) take tens of minutes.
|
||||
static bool TransferBlocks(uint64_t lba, uint32_t count, uint8_t* data, bool write) {
|
||||
uint32_t address = g_highCapacity ? (uint32_t)lba : (uint32_t)(lba * BlockSize);
|
||||
bool multi = count > 1;
|
||||
Write(REG_BLOCK_SIZE_COUNT, count << 16 | BlockSize);
|
||||
|
||||
uint32_t mode = write ? 0 : XM_READ;
|
||||
if (multi) mode |= XM_BLOCK_COUNT_EN | XM_AUTO_CMD12 | XM_MULTI_BLOCK;
|
||||
uint32_t command = write ? (multi ? 25 : 24) : (multi ? 18 : 17);
|
||||
|
||||
bool ok = SendCommand(command, address, Resp48, mode, true);
|
||||
for (uint32_t i = 0; ok && i < count; i++) {
|
||||
ok = WaitStatus(write ? INT_WRITE_READY : INT_READ_READY, DataTimeoutMs);
|
||||
if (ok) PioBlock(data + (uint64_t)i * BlockSize, write);
|
||||
}
|
||||
ok = ok && WaitStatus(INT_XFER_COMPLETE, DataTimeoutMs);
|
||||
|
||||
// A failed multi-block transfer leaves the card sending or receiving
|
||||
// data; stop it so the next command finds it in the transfer state.
|
||||
if (!ok && multi) SendCommand(12, 0, Resp48Busy);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool TransferSectors(uint64_t lba, uint32_t count, uint8_t* data, bool write) {
|
||||
if (lba + count > g_sectorCount) return false;
|
||||
g_lock.Acquire();
|
||||
bool ok = true;
|
||||
while (ok && count > 0) {
|
||||
uint32_t run = count > MaxBlocksPerCommand ? MaxBlocksPerCommand : count;
|
||||
ok = TransferBlocks(lba, run, data, write);
|
||||
lba += run;
|
||||
data += (uint64_t)run * BlockSize;
|
||||
count -= run;
|
||||
}
|
||||
g_lock.Release();
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool ReadSectors(void*, uint64_t lba, uint32_t count, void* buffer) {
|
||||
return TransferSectors(lba, count, (uint8_t*)buffer, false);
|
||||
}
|
||||
|
||||
static bool WriteSectors(void*, uint64_t lba, uint32_t count, const void* buffer) {
|
||||
return TransferSectors(lba, count, (uint8_t*)buffer, true);
|
||||
}
|
||||
|
||||
static bool InitializeController(uint64_t mmioPhys, const char* name) {
|
||||
if (g_initialized) return false;
|
||||
|
||||
Memory::VMM::g_paging->MapMMIO(mmioPhys & ~0xFFFULL, Memory::HHDM(mmioPhys & ~0xFFFULL));
|
||||
g_regs = (volatile uint8_t*)Memory::HHDM(mmioPhys);
|
||||
|
||||
if (!Reset(RESET_ALL)) {
|
||||
KernelLogStream(ERROR, "SDHCI") << name << ": reset timed out";
|
||||
return false;
|
||||
}
|
||||
|
||||
g_version = (Read(REG_VERSION) >> 16) & 0xFF;
|
||||
uint32_t caps = Read(REG_CAPABILITIES);
|
||||
uint32_t baseMhz = g_version >= 2 ? (caps >> 8) & 0xFF : (caps >> 8) & 0x3F;
|
||||
g_baseClockHz = baseMhz != 0 ? baseMhz * 1000000 : FallbackBaseClockHz;
|
||||
|
||||
uint32_t voltage = (caps & CAP_VOLT_330) ? PWR_330 : (caps & CAP_VOLT_300) ? PWR_300 : PWR_180;
|
||||
Write(REG_HOST_CONTROL, (Read(REG_HOST_CONTROL) & ~0xFF00u) | voltage | PWR_ON);
|
||||
|
||||
Write(REG_INT_ENABLE, 0xFFFFFFFF);
|
||||
Write(REG_INT_SIGNAL, 0);
|
||||
Write(REG_CLOCK_CONTROL, (Read(REG_CLOCK_CONTROL) & ~0xFF0000u) | TIMEOUT_MAX);
|
||||
|
||||
if (!SetClock(400000)) {
|
||||
KernelLogStream(ERROR, "SDHCI") << name << ": clock did not stabilize";
|
||||
return false;
|
||||
}
|
||||
|
||||
// The card needs 74 clocks after power-up before its first command.
|
||||
uint64_t start = Timekeeping::GetMilliseconds();
|
||||
while (Timekeeping::GetMilliseconds() - start < 2) Arch::Pause();
|
||||
|
||||
KernelLogStream(INFO, "SDHCI") << name << ": version " << base::dec
|
||||
<< (uint64_t)(g_version + 1) << ".00, base clock " << (uint64_t)(g_baseClockHz / 1000000)
|
||||
<< " MHz" << (baseMhz == 0 ? " (assumed)" : "");
|
||||
|
||||
if (!InitializeCard()) {
|
||||
KernelLogStream(INFO, "SDHCI") << name << ": no SD card";
|
||||
return false;
|
||||
}
|
||||
|
||||
g_initialized = true;
|
||||
KernelLogStream(OK, "SDHCI") << name << ": " << (g_highCapacity ? "SDHC/SDXC" : "SDSC")
|
||||
<< " card, " << base::dec << (g_sectorCount * BlockSize / (1024 * 1024)) << " MiB";
|
||||
|
||||
BlockDevice dev = {};
|
||||
dev.ReadSectors = ReadSectors;
|
||||
dev.WriteSectors = WriteSectors;
|
||||
dev.SectorCount = g_sectorCount;
|
||||
dev.SectorSize = BlockSize;
|
||||
dev.Kind = BLOCK_KIND_SD;
|
||||
const char model[] = "SD card";
|
||||
memcpy(dev.Model, model, sizeof(model));
|
||||
RegisterBlockDevice(dev);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Probe(const Pci::PciDevice& dev) {
|
||||
uint64_t bar = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
|
||||
if (bar == 0) return false;
|
||||
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
|
||||
return InitializeController(bar, "PCI controller");
|
||||
}
|
||||
|
||||
bool ProbePlatform(uint64_t mmioPhys, const char* name) {
|
||||
return InitializeController(mmioPhys, name);
|
||||
}
|
||||
|
||||
bool IsInitialized() {
|
||||
return g_initialized;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Sdhci.hpp
|
||||
* SD Host Controller (SDHCI) driver: SD cards as block devices
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <Pci/Pci.hpp>
|
||||
|
||||
namespace Drivers::Storage::Sdhci {
|
||||
|
||||
// A standard SD host controller on PCI (class 08h/05h).
|
||||
bool Probe(const Pci::PciDevice& dev);
|
||||
|
||||
// One ACPI describes (the Raspberry Pi 4's EMMC2, BRCME88C; a generic
|
||||
// PNP0D40 controller): its register block.
|
||||
bool ProbePlatform(uint64_t mmioPhys, const char* name);
|
||||
|
||||
bool IsInitialized();
|
||||
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
/*
|
||||
* Hub.cpp
|
||||
* USB hub class driver (USB 2.0 and SuperSpeed hubs)
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#include "Hub.hpp"
|
||||
#include "Xhci.hpp"
|
||||
#include "UsbDevice.hpp"
|
||||
#include <Arch/Cpu.hpp>
|
||||
#include <Terminal/Terminal.hpp>
|
||||
#include <CppLib/Stream.hpp>
|
||||
#include <Timekeeping/Timer.hpp>
|
||||
#include <atomic>
|
||||
|
||||
using namespace Kt;
|
||||
|
||||
static void BusyWaitMs(uint64_t ms) {
|
||||
if (Arch::InterruptsEnabled()) {
|
||||
uint64_t start = Timekeeping::GetMilliseconds();
|
||||
while (Timekeeping::GetMilliseconds() - start < ms) {
|
||||
Arch::Pause();
|
||||
}
|
||||
} else {
|
||||
// The tick is not advancing: count ~1 us delays instead
|
||||
for (uint64_t i = 0; i < ms * 1000; i++) {
|
||||
Arch::IoDelay();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace Drivers::USB::Hub {
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Hub class requests and port status (USB 2.0 11.24, USB 3.2 10.16)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
constexpr uint8_t REQTYPE_HUB_IN = 0xA0; // Dev-to-host, class, device
|
||||
constexpr uint8_t REQTYPE_HUB_OUT = 0x20; // Host-to-dev, class, device
|
||||
constexpr uint8_t REQTYPE_PORT_IN = 0xA3; // Dev-to-host, class, other (port)
|
||||
constexpr uint8_t REQTYPE_PORT_OUT = 0x23; // Host-to-dev, class, other (port)
|
||||
|
||||
constexpr uint8_t REQ_GET_STATUS = 0x00;
|
||||
constexpr uint8_t REQ_CLEAR_FEATURE = 0x01;
|
||||
constexpr uint8_t REQ_SET_FEATURE = 0x03;
|
||||
constexpr uint8_t REQ_GET_DESCRIPTOR = 0x06;
|
||||
constexpr uint8_t REQ_SET_HUB_DEPTH = 0x0C; // SuperSpeed hubs only
|
||||
|
||||
constexpr uint8_t DESC_HUB = 0x29;
|
||||
constexpr uint8_t DESC_SS_HUB = 0x2A;
|
||||
|
||||
// Port feature selectors
|
||||
constexpr uint16_t FEAT_PORT_RESET = 4;
|
||||
constexpr uint16_t FEAT_PORT_POWER = 8;
|
||||
|
||||
// wPortStatus
|
||||
constexpr uint16_t PS_CONNECTION = (1 << 0);
|
||||
constexpr uint16_t PS_ENABLE = (1 << 1);
|
||||
constexpr uint16_t PS_RESET = (1 << 4);
|
||||
constexpr uint16_t PS_LOW_SPEED = (1 << 9); // USB 2 hubs
|
||||
constexpr uint16_t PS_HIGH_SPEED = (1 << 10); // USB 2 hubs
|
||||
|
||||
// wPortChange
|
||||
constexpr uint16_t PC_CONNECTION = (1 << 0);
|
||||
constexpr uint16_t PC_RESET = (1 << 4);
|
||||
|
||||
// The route string holds a 4-bit port number per tier, five tiers.
|
||||
constexpr uint8_t MAX_TIERS = 5;
|
||||
constexpr uint8_t MAX_ROUTABLE_PORT = 15;
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// State
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
struct HubState {
|
||||
bool Active;
|
||||
HubDescriptorInfo Info;
|
||||
};
|
||||
|
||||
static HubState g_hubs[Xhci::MAX_SLOTS + 1] = {};
|
||||
|
||||
// Written from the event-ring drain, consumed by ProcessPendingWork.
|
||||
static std::atomic<uint32_t> g_changedPorts[Xhci::MAX_SLOTS + 1] = {}; // bit n = port n
|
||||
static std::atomic<bool> g_rearm[Xhci::MAX_SLOTS + 1] = {};
|
||||
static std::atomic<bool> g_endpointFailed[Xhci::MAX_SLOTS + 1] = {};
|
||||
static std::atomic<bool> g_pending{false};
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Requests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
static bool Succeeded(uint32_t cc) {
|
||||
return cc == Xhci::CC_SUCCESS || cc == Xhci::CC_SHORT_PACKET;
|
||||
}
|
||||
|
||||
static bool GetPortStatus(uint8_t hubSlot, uint8_t port, uint16_t& status, uint16_t& change) {
|
||||
uint8_t buf[4] = {};
|
||||
uint32_t cc = Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_IN, REQ_GET_STATUS,
|
||||
0, port, sizeof(buf), buf, true);
|
||||
if (!Succeeded(cc)) return false;
|
||||
status = (uint16_t)(buf[0] | (buf[1] << 8));
|
||||
change = (uint16_t)(buf[2] | (buf[3] << 8));
|
||||
return true;
|
||||
}
|
||||
|
||||
static uint32_t SetPortFeature(uint8_t hubSlot, uint8_t port, uint16_t feature) {
|
||||
return Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_OUT, REQ_SET_FEATURE,
|
||||
feature, port, 0, nullptr, false);
|
||||
}
|
||||
|
||||
static uint32_t ClearPortFeature(uint8_t hubSlot, uint8_t port, uint16_t feature) {
|
||||
return Xhci::ControlTransfer(hubSlot, REQTYPE_PORT_OUT, REQ_CLEAR_FEATURE,
|
||||
feature, port, 0, nullptr, false);
|
||||
}
|
||||
|
||||
// Acknowledge every change bit in `change`. Bits 0-4 clear with
|
||||
// C_PORT_CONNECTION..C_PORT_RESET (16-20); SuperSpeed hubs add
|
||||
// C_BH_PORT_RESET, C_PORT_LINK_STATE and C_PORT_CONFIG_ERROR.
|
||||
static void ClearPortChanges(uint8_t hubSlot, uint8_t port, uint16_t change, bool superSpeed) {
|
||||
for (uint16_t bit = 0; bit <= 4; bit++) {
|
||||
if (change & (1u << bit)) ClearPortFeature(hubSlot, port, (uint16_t)(16 + bit));
|
||||
}
|
||||
if (superSpeed) {
|
||||
if (change & (1u << 5)) ClearPortFeature(hubSlot, port, 29);
|
||||
if (change & (1u << 6)) ClearPortFeature(hubSlot, port, 25);
|
||||
if (change & (1u << 7)) ClearPortFeature(hubSlot, port, 26);
|
||||
}
|
||||
}
|
||||
|
||||
// Acknowledge hub-wide changes (local power, over-current): until they
|
||||
// are cleared the hub keeps reporting bit 0 of its change bitmap.
|
||||
static void ClearHubChanges(uint8_t hubSlot) {
|
||||
uint8_t buf[4] = {};
|
||||
uint32_t cc = Xhci::ControlTransfer(hubSlot, REQTYPE_HUB_IN, REQ_GET_STATUS,
|
||||
0, 0, sizeof(buf), buf, true);
|
||||
if (!Succeeded(cc)) return;
|
||||
uint16_t change = (uint16_t)(buf[2] | (buf[3] << 8));
|
||||
for (uint16_t bit = 0; bit <= 1; bit++) { // C_HUB_LOCAL_POWER, C_HUB_OVER_CURRENT
|
||||
if (change & (1u << bit)) {
|
||||
Xhci::ControlTransfer(hubSlot, REQTYPE_HUB_OUT, REQ_CLEAR_FEATURE,
|
||||
bit, 0, 0, nullptr, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool ReadDescriptor(uint8_t slotId, uint32_t speed, HubDescriptorInfo& out) {
|
||||
out = {};
|
||||
out.SuperSpeed = speed >= Xhci::SPEED_SUPER;
|
||||
uint8_t type = out.SuperSpeed ? DESC_SS_HUB : DESC_HUB;
|
||||
|
||||
uint8_t buf[16] = {};
|
||||
uint32_t cc = Xhci::ControlTransfer(slotId, REQTYPE_HUB_IN, REQ_GET_DESCRIPTOR,
|
||||
(uint16_t)(type << 8), 0, sizeof(buf), buf, true);
|
||||
if (!Succeeded(cc) || buf[0] < 7 || buf[1] != type || buf[2] == 0) {
|
||||
KernelLogStream(ERROR, "USB/Hub") << "Slot " << base::dec << (uint64_t)slotId
|
||||
<< ": hub descriptor unreadable, cc=" << (uint64_t)cc;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint16_t characteristics = (uint16_t)(buf[3] | (buf[4] << 8));
|
||||
out.Ports = buf[2];
|
||||
out.ThinkTime = out.SuperSpeed ? 0 : (uint8_t)((characteristics >> 5) & 3);
|
||||
out.PowerOnMs = (uint16_t)(buf[5] * 2);
|
||||
return true;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Ports
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
static uint8_t FindChild(uint8_t hubSlot, uint8_t port) {
|
||||
for (uint8_t s = 1; s <= Xhci::MAX_SLOTS; s++) {
|
||||
auto* dev = Xhci::GetDevice(s);
|
||||
if (dev->Active && dev->ParentSlot == hubSlot && dev->ParentPort == port) return s;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Reset a connected port and report the speed the device came up at.
|
||||
static bool ResetPort(uint8_t hubSlot, uint8_t port, bool superSpeed, uint32_t& speed) {
|
||||
if (SetPortFeature(hubSlot, port, FEAT_PORT_RESET) != Xhci::CC_SUCCESS) return false;
|
||||
|
||||
uint16_t status = 0, change = 0;
|
||||
bool done = false;
|
||||
for (int attempt = 0; attempt < 50; attempt++) { // ~500 ms
|
||||
BusyWaitMs(10);
|
||||
if (!GetPortStatus(hubSlot, port, status, change)) return false;
|
||||
if ((change & PC_RESET) && !(status & PS_RESET)) { done = true; break; }
|
||||
if (!(status & PS_CONNECTION)) return false;
|
||||
}
|
||||
ClearPortChanges(hubSlot, port, change, superSpeed);
|
||||
if (!done || !(status & PS_ENABLE)) return false;
|
||||
|
||||
if (superSpeed) speed = Xhci::SPEED_SUPER;
|
||||
else if (status & PS_LOW_SPEED) speed = Xhci::SPEED_LOW;
|
||||
else if (status & PS_HIGH_SPEED) speed = Xhci::SPEED_HIGH;
|
||||
else speed = Xhci::SPEED_FULL;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Bring the port's state in line with what is attached to it: remove a
|
||||
// device that left (or was replaced), enumerate one that arrived.
|
||||
static void HandlePort(uint8_t hubSlot, uint8_t port, bool settled) {
|
||||
const HubDescriptorInfo& info = g_hubs[hubSlot].Info;
|
||||
|
||||
uint16_t status = 0, change = 0;
|
||||
if (!GetPortStatus(hubSlot, port, status, change)) return;
|
||||
if (change != 0) ClearPortChanges(hubSlot, port, change, info.SuperSpeed);
|
||||
|
||||
bool connected = (status & PS_CONNECTION) != 0;
|
||||
uint8_t child = FindChild(hubSlot, port);
|
||||
if (child != 0 && (!connected || (change & PC_CONNECTION))) {
|
||||
KernelLogStream(INFO, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
|
||||
<< " port " << (uint64_t)port << ": device disconnected";
|
||||
Xhci::RemoveDevice(child);
|
||||
child = 0;
|
||||
}
|
||||
if (!connected || child != 0) return;
|
||||
|
||||
if (!settled) {
|
||||
// Connect debounce (USB 2.0 7.1.7.3): let contacts settle first.
|
||||
BusyWaitMs(100);
|
||||
if (!GetPortStatus(hubSlot, port, status, change)) return;
|
||||
if (!(status & PS_CONNECTION)) return;
|
||||
}
|
||||
|
||||
auto* hub = Xhci::GetDevice(hubSlot);
|
||||
if (hub->Depth + 1 > MAX_TIERS || port > MAX_ROUTABLE_PORT) {
|
||||
KernelLogStream(WARNING, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
|
||||
<< " port " << (uint64_t)port << ": too deep to address, ignored";
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t speed = 0;
|
||||
if (!ResetPort(hubSlot, port, info.SuperSpeed, speed)) {
|
||||
KernelLogStream(WARNING, "USB/Hub") << "Slot " << base::dec << (uint64_t)hubSlot
|
||||
<< " port " << (uint64_t)port << ": reset failed";
|
||||
return;
|
||||
}
|
||||
BusyWaitMs(10); // reset recovery (USB 2.0 7.1.7.5)
|
||||
|
||||
UsbDevice::DeviceLocation location = {};
|
||||
location.RootPort = hub->PortId;
|
||||
location.ParentSlot = hubSlot;
|
||||
location.ParentPort = port;
|
||||
location.Depth = (uint8_t)(hub->Depth + 1);
|
||||
location.RouteString = hub->RouteString | ((uint32_t)port << (4 * hub->Depth));
|
||||
if (hub->Speed == Xhci::SPEED_HIGH &&
|
||||
(speed == Xhci::SPEED_LOW || speed == Xhci::SPEED_FULL)) {
|
||||
// This hub's transaction translator turns the high-speed
|
||||
// traffic into low/full-speed transactions. Single-TT mode:
|
||||
// the configuration's default alternate setting.
|
||||
location.TtHubSlot = hubSlot;
|
||||
location.TtPort = port;
|
||||
location.TtMulti = false;
|
||||
} else {
|
||||
location.TtHubSlot = hub->TtHubSlot;
|
||||
location.TtPort = hub->TtPort;
|
||||
location.TtMulti = hub->TtMulti;
|
||||
}
|
||||
|
||||
UsbDevice::EnumerateDevice(location, speed);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Public API
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
void RegisterHub(uint8_t slotId, const HubDescriptorInfo& info) {
|
||||
auto* dev = Xhci::GetDevice(slotId);
|
||||
if (dev == nullptr || !dev->Active) return;
|
||||
|
||||
g_hubs[slotId].Info = info;
|
||||
g_hubs[slotId].Active = true;
|
||||
g_changedPorts[slotId].store(0, std::memory_order_relaxed);
|
||||
g_rearm[slotId].store(false, std::memory_order_relaxed);
|
||||
g_endpointFailed[slotId].store(false, std::memory_order_relaxed);
|
||||
|
||||
uint8_t ports = info.Ports;
|
||||
if (ports > 31) ports = 31; // status-change bitmap tracked in 32 bits
|
||||
|
||||
KernelLogStream(OK, "USB/Hub") << "Slot " << base::dec << (uint64_t)slotId << ": "
|
||||
<< (info.SuperSpeed ? "SuperSpeed" : "USB 2.0") << " hub, "
|
||||
<< (uint64_t)info.Ports << " ports, tier " << (uint64_t)(dev->Depth + 1);
|
||||
|
||||
if (info.SuperSpeed) {
|
||||
Xhci::ControlTransfer(slotId, REQTYPE_HUB_OUT, REQ_SET_HUB_DEPTH,
|
||||
dev->Depth, 0, 0, nullptr, false);
|
||||
}
|
||||
|
||||
for (uint8_t port = 1; port <= ports; port++) {
|
||||
SetPortFeature(slotId, port, FEAT_PORT_POWER);
|
||||
}
|
||||
// Power-on-to-power-good, then the connect debounce interval.
|
||||
uint64_t wait = info.PowerOnMs < 100 ? 100 : info.PowerOnMs;
|
||||
BusyWaitMs(wait + 100);
|
||||
|
||||
for (uint8_t port = 1; port <= ports && g_hubs[slotId].Active; port++) {
|
||||
HandlePort(slotId, port, true);
|
||||
}
|
||||
|
||||
if (g_hubs[slotId].Active) Xhci::QueueInterruptTransfer(slotId);
|
||||
}
|
||||
|
||||
void UnregisterHub(uint8_t slotId) {
|
||||
if (slotId == 0 || slotId > Xhci::MAX_SLOTS) return;
|
||||
g_hubs[slotId] = {};
|
||||
g_changedPorts[slotId].store(0, std::memory_order_relaxed);
|
||||
g_rearm[slotId].store(false, std::memory_order_relaxed);
|
||||
g_endpointFailed[slotId].store(false, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void OnStatusChange(uint8_t slotId, const uint8_t* data, uint16_t length,
|
||||
uint32_t completionCode) {
|
||||
if (slotId == 0 || slotId > Xhci::MAX_SLOTS) return;
|
||||
|
||||
if (Succeeded(completionCode)) {
|
||||
uint32_t bits = 0;
|
||||
for (uint16_t i = 0; i < length && i < 4; i++) {
|
||||
bits |= (uint32_t)data[i] << (8 * i);
|
||||
}
|
||||
g_changedPorts[slotId].fetch_or(bits, std::memory_order_relaxed);
|
||||
g_rearm[slotId].store(true, std::memory_order_relaxed);
|
||||
} else {
|
||||
// The endpoint halted; it needs a reset before it reports again,
|
||||
// and changes may have been missed meanwhile.
|
||||
g_endpointFailed[slotId].store(true, std::memory_order_relaxed);
|
||||
}
|
||||
g_pending.store(true, std::memory_order_release);
|
||||
}
|
||||
|
||||
void RequestPortCheck(uint8_t hubSlot, uint8_t port) {
|
||||
if (hubSlot == 0 || hubSlot > Xhci::MAX_SLOTS || port == 0 || port > 31) return;
|
||||
g_changedPorts[hubSlot].fetch_or(1u << port, std::memory_order_relaxed);
|
||||
g_pending.store(true, std::memory_order_release);
|
||||
}
|
||||
|
||||
bool HasPendingWork() {
|
||||
return g_pending.load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
void ProcessPendingWork() {
|
||||
if (!g_pending.exchange(false, std::memory_order_acq_rel)) return;
|
||||
|
||||
for (uint8_t slot = 1; slot <= Xhci::MAX_SLOTS; slot++) {
|
||||
if (!g_hubs[slot].Active) continue;
|
||||
|
||||
bool failed = g_endpointFailed[slot].exchange(false, std::memory_order_acq_rel);
|
||||
bool rearm = g_rearm[slot].exchange(false, std::memory_order_acq_rel);
|
||||
uint32_t changed = g_changedPorts[slot].exchange(0, std::memory_order_acq_rel);
|
||||
if (!failed && !rearm && changed == 0) continue;
|
||||
|
||||
uint8_t ports = g_hubs[slot].Info.Ports;
|
||||
if (ports > 31) ports = 31;
|
||||
|
||||
if (failed) {
|
||||
// Reset Endpoint re-arms the endpoint itself.
|
||||
Xhci::ResetInterruptEndpoint(slot);
|
||||
changed = ((ports == 31) ? 0xFFFFFFFFu : ((1u << (ports + 1)) - 1)) & ~1u;
|
||||
}
|
||||
|
||||
if (changed & 1u) ClearHubChanges(slot);
|
||||
for (uint8_t port = 1; port <= ports && g_hubs[slot].Active; port++) {
|
||||
if (changed & (1u << port)) HandlePort(slot, port, false);
|
||||
}
|
||||
|
||||
if (rearm && !failed && g_hubs[slot].Active) Xhci::QueueInterruptTransfer(slot);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Hub.hpp
|
||||
* USB hub class driver (USB 2.0 and SuperSpeed hubs)
|
||||
* Copyright (c) 2026 Daniel Hammer
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
// Devices behind a hub get their own xHCI slots, addressed by route string
|
||||
// (Xhci::UsbDeviceInfo). The hub driver powers and resets the hub's ports
|
||||
// and enumerates what it finds; the hub's interrupt endpoint reports port
|
||||
// changes afterwards, handled from the xHCI deferred-work pass.
|
||||
//
|
||||
// The Raspberry Pi 4/400 needs this for any USB 2 device: the VL805's only
|
||||
// USB 2 root port carries an on-board VIA hub that every USB 2 line (the
|
||||
// Pi 400's built-in keyboard included) hangs off.
|
||||
|
||||
namespace Drivers::USB::Hub {
|
||||
|
||||
struct HubDescriptorInfo {
|
||||
uint8_t Ports;
|
||||
uint8_t ThinkTime; // TT think time code, wHubCharacteristics[6:5]
|
||||
uint16_t PowerOnMs; // bPwrOn2PwrGood, in ms
|
||||
bool SuperSpeed;
|
||||
};
|
||||
|
||||
// Read the hub class descriptor of a configured hub. `speed` is the
|
||||
// hub's xHCI port speed (SuperSpeed hubs have their own descriptor).
|
||||
bool ReadDescriptor(uint8_t slotId, uint32_t speed, HubDescriptorInfo& out);
|
||||
|
||||
// Take over a hub whose slot is configured as a hub with its interrupt
|
||||
// endpoint: power its ports, enumerate the devices on them, then arm the
|
||||
// status-change endpoint. Process context.
|
||||
void RegisterHub(uint8_t slotId, const HubDescriptorInfo& info);
|
||||
|
||||
// Forget a hub (its children are removed by Xhci::RemoveDevice).
|
||||
void UnregisterHub(uint8_t slotId);
|
||||
|
||||
// A completion on the hub's status-change endpoint. Called from the
|
||||
// event ring drain: records the change and nothing else.
|
||||
void OnStatusChange(uint8_t slotId, const uint8_t* data, uint16_t length,
|
||||
uint32_t completionCode);
|
||||
|
||||
// Look at a port again on the next ProcessPendingWork as if it had
|
||||
// reported a change: a device removed from it is reset and enumerated
|
||||
// afresh, as if replugged.
|
||||
void RequestPortCheck(uint8_t hubSlot, uint8_t port);
|
||||
|
||||
// Port changes are waiting for ProcessPendingWork.
|
||||
bool HasPendingWork();
|
||||
|
||||
// Handle recorded port changes and re-arm the endpoints. Runs inside the
|
||||
// xHCI deferred-work pass (process/idle context, single runner).
|
||||
void ProcessPendingWork();
|
||||
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "HidKeyboard.hpp"
|
||||
#include "HidMouse.hpp"
|
||||
#include "MassStorage.hpp"
|
||||
#include "Hub.hpp"
|
||||
#include "Bluetooth/Bluetooth.hpp"
|
||||
#include <Terminal/Terminal.hpp>
|
||||
#include <CppLib/Stream.hpp>
|
||||
@@ -65,24 +66,28 @@ namespace Drivers::USB::UsbDevice {
|
||||
return speed; // Same encoding
|
||||
}
|
||||
|
||||
// Convert USB endpoint bInterval to xHCI Endpoint Context Interval value.
|
||||
// HS/SS: bInterval is already in 2^(n-1) * 125µs encoding — use directly.
|
||||
// FS/LS: bInterval is in milliseconds (frames) — convert via fls(bInterval * 8).
|
||||
// Convert USB endpoint bInterval to xHCI Endpoint Context Interval value
|
||||
// (a period of 2^Interval * 125 us), as Linux's xhci_get_endpoint_interval.
|
||||
// HS/SS: bInterval encodes a period of 2^(bInterval-1) microframes.
|
||||
// FS/LS: bInterval is in milliseconds (frames); round down to a power of
|
||||
// two microframes, between 1 ms and 128 ms.
|
||||
static uint32_t ConvertInterval(uint32_t speed, uint8_t bInterval) {
|
||||
if (bInterval == 0) return 0;
|
||||
|
||||
if (speed == Xhci::SPEED_HIGH || speed == Xhci::SPEED_SUPER) {
|
||||
return bInterval;
|
||||
uint32_t b = bInterval;
|
||||
if (b < 1) b = 1;
|
||||
if (b > 16) b = 16;
|
||||
return b - 1;
|
||||
}
|
||||
|
||||
// FS/LS: bInterval ms → microframes, then find highest set bit position
|
||||
if (bInterval == 0) return 3;
|
||||
uint32_t microframes = (uint32_t)bInterval * 8;
|
||||
uint32_t interval = 0;
|
||||
while (microframes > 0) {
|
||||
while (microframes > 1) {
|
||||
interval++;
|
||||
microframes >>= 1;
|
||||
}
|
||||
if (interval > 15) interval = 15;
|
||||
if (interval < 3) interval = 3;
|
||||
if (interval > 10) interval = 10;
|
||||
return interval;
|
||||
}
|
||||
|
||||
@@ -101,9 +106,25 @@ namespace Drivers::USB::UsbDevice {
|
||||
// EnumerateDevice
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static bool EnumerateInSlot(uint8_t slotId, const DeviceLocation& location, uint32_t speed,
|
||||
bool& isHub, Hub::HubDescriptorInfo& hubInfo);
|
||||
|
||||
uint8_t EnumerateDevice(uint8_t portId, uint32_t speed) {
|
||||
KernelLogStream(INFO, "USB") << "Enumerating device on port "
|
||||
<< (uint64_t)portId << " speed=" << SpeedToString(speed);
|
||||
DeviceLocation location = {};
|
||||
location.RootPort = portId;
|
||||
return EnumerateDevice(location, speed);
|
||||
}
|
||||
|
||||
uint8_t EnumerateDevice(const DeviceLocation& location, uint32_t speed) {
|
||||
if (location.ParentSlot == 0) {
|
||||
KernelLogStream(INFO, "USB") << "Enumerating device on port "
|
||||
<< (uint64_t)location.RootPort << " speed=" << SpeedToString(speed);
|
||||
} else {
|
||||
KernelLogStream(INFO, "USB") << "Enumerating device on hub slot "
|
||||
<< (uint64_t)location.ParentSlot << " port " << (uint64_t)location.ParentPort
|
||||
<< " (route " << base::hex << (uint64_t)location.RouteString << base::dec
|
||||
<< ", root port " << (uint64_t)location.RootPort << ") speed=" << SpeedToString(speed);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 1: Enable Slot
|
||||
@@ -125,6 +146,31 @@ namespace Drivers::USB::UsbDevice {
|
||||
|
||||
KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId << " enabled";
|
||||
|
||||
bool isHub = false;
|
||||
Hub::HubDescriptorInfo hubInfo = {};
|
||||
if (!EnumerateInSlot(slotId, location, speed, isHub, hubInfo)) {
|
||||
// Give the slot back, or every failed attempt (a flaky device
|
||||
// re-reported by its hub) would leak one until none are left.
|
||||
Xhci::TRB disableTrb = {};
|
||||
disableTrb.Control = (Xhci::TRB_DISABLE_SLOT << Xhci::TRB_TYPE_SHIFT)
|
||||
| ((uint32_t)slotId << 24);
|
||||
Xhci::SendCommand(disableTrb);
|
||||
Xhci::g_dcbaa[slotId] = 0;
|
||||
*Xhci::GetDevice(slotId) = {};
|
||||
return 0;
|
||||
}
|
||||
|
||||
// A hub enumerates what is behind it now that its own slot is
|
||||
// published (its children name it as their parent).
|
||||
if (isHub) Hub::RegisterHub(slotId, hubInfo);
|
||||
return slotId;
|
||||
}
|
||||
|
||||
static bool EnumerateInSlot(uint8_t slotId, const DeviceLocation& location, uint32_t speed,
|
||||
bool& isHub, Hub::HubDescriptorInfo& hubInfo) {
|
||||
uint8_t portId = location.RootPort;
|
||||
uint32_t cc = 0;
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 2: Allocate device output context and set DCBAA entry
|
||||
// -----------------------------------------------------------------
|
||||
@@ -133,9 +179,16 @@ namespace Drivers::USB::UsbDevice {
|
||||
dev->Active = true;
|
||||
dev->PortId = portId;
|
||||
dev->Speed = speed;
|
||||
dev->ParentSlot = location.ParentSlot;
|
||||
dev->ParentPort = location.ParentPort;
|
||||
dev->Depth = location.Depth;
|
||||
dev->RouteString = location.RouteString;
|
||||
dev->TtHubSlot = location.TtHubSlot;
|
||||
dev->TtPort = location.TtPort;
|
||||
dev->TtMulti = location.TtMulti;
|
||||
|
||||
// Allocate a zeroed page for the output DeviceContext
|
||||
auto* outputCtx = (Xhci::DeviceContext*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* outputCtx = (Xhci::DeviceContext*)Xhci::AllocateDmaPage();
|
||||
dev->OutputContext = outputCtx;
|
||||
dev->OutputContextPhys = Memory::SubHHDM(outputCtx);
|
||||
|
||||
@@ -145,7 +198,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
// -----------------------------------------------------------------
|
||||
// Step 3: Build Input Context for Address Device command
|
||||
// -----------------------------------------------------------------
|
||||
auto* inputCtx = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* inputCtx = (Xhci::InputContext*)Xhci::AllocateDmaPage();
|
||||
|
||||
// Input Control Context: add Slot Context (bit 0) and EP0 (bit 1)
|
||||
inputCtx->ICC.AddFlags = 0x3;
|
||||
@@ -153,12 +206,18 @@ namespace Drivers::USB::UsbDevice {
|
||||
// Slot Context
|
||||
uint32_t speedVal = SpeedToSlotContextValue(speed);
|
||||
uint32_t ctxEntries = 1; // Context Entries = 1 (Slot + EP0 only)
|
||||
inputCtx->Slot.Field0 = (ctxEntries << 27) | (speedVal << 20);
|
||||
inputCtx->Slot.Field0 = (ctxEntries << 27) | (speedVal << 20)
|
||||
| (location.RouteString & 0xFFFFF)
|
||||
| (location.TtMulti ? Xhci::SLOT_MTT : 0);
|
||||
inputCtx->Slot.Field1 = ((uint32_t)portId << 16); // Root Hub Port Number
|
||||
// Low/full-speed behind a high-speed hub: the hub slot and port
|
||||
// whose transaction translator the controller addresses.
|
||||
inputCtx->Slot.Field2 = (uint32_t)location.TtHubSlot
|
||||
| ((uint32_t)location.TtPort << 8);
|
||||
|
||||
// EP0 Context
|
||||
// Allocate EP0 transfer ring
|
||||
auto* ep0Ring = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* ep0Ring = (Xhci::TRB*)Xhci::AllocateDmaPage();
|
||||
dev->EP0Ring = ep0Ring;
|
||||
dev->EP0RingPhys = Memory::SubHHDM(ep0Ring);
|
||||
dev->EP0RingEnqueue = 0;
|
||||
@@ -198,8 +257,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
if (cc != Xhci::CC_SUCCESS) {
|
||||
KernelLogStream(ERROR, "USB") << "Address Device (BSR=1) failed, slot="
|
||||
<< (uint64_t)slotId << " cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
KernelLogStream(INFO, "USB") << "Slot " << (uint64_t)slotId << " initialized (BSR=1)";
|
||||
@@ -213,8 +271,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
partialDesc, true);
|
||||
if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
|
||||
KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(8-byte) failed, cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint8_t bMaxPacketSize0 = partialDesc[7];
|
||||
@@ -227,7 +284,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
// Step 4c: Evaluate Context — update EP0 max packet size if needed
|
||||
// -----------------------------------------------------------------
|
||||
if (bMaxPacketSize0 != maxPacket) {
|
||||
auto* evalCtx = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* evalCtx = (Xhci::InputContext*)Xhci::AllocateDmaPage();
|
||||
|
||||
// Only updating EP0 — set AddFlags bit 1 (EP0), no slot context needed
|
||||
evalCtx->ICC.AddFlags = (1 << 1);
|
||||
@@ -280,8 +337,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
if (cc != Xhci::CC_SUCCESS) {
|
||||
KernelLogStream(ERROR, "USB") << "Address Device failed, slot="
|
||||
<< (uint64_t)slotId << " cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Set-address recovery time (USB spec requires >= 2ms, use 10ms for safety)
|
||||
@@ -298,8 +354,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
&devDesc, true);
|
||||
if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
|
||||
KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Device) failed, cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
dev->VendorId = devDesc.idVendor;
|
||||
@@ -319,8 +374,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
&cfgHdr, true);
|
||||
if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
|
||||
KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Config header) failed, cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
uint8_t cfgBuf[256] = {};
|
||||
@@ -332,8 +386,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
cfgBuf, true);
|
||||
if (cc != Xhci::CC_SUCCESS && cc != Xhci::CC_SHORT_PACKET) {
|
||||
KernelLogStream(ERROR, "USB") << "GET_DESCRIPTOR(Config full) failed, cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
@@ -341,6 +394,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
// -----------------------------------------------------------------
|
||||
uint16_t offset = 0;
|
||||
bool foundHid = false;
|
||||
bool currentHub = false;
|
||||
bool foundBt = false;
|
||||
bool foundMsc = false;
|
||||
bool currentMsc = false;
|
||||
@@ -358,9 +412,23 @@ namespace Drivers::USB::UsbDevice {
|
||||
auto* iface = (InterfaceDescriptor*)&cfgBuf[offset];
|
||||
// Reset at each new interface boundary
|
||||
foundHid = false;
|
||||
currentHub = false;
|
||||
foundBt = false;
|
||||
currentMsc = false;
|
||||
|
||||
// Hub (class 9): its status-change interrupt endpoint. Only
|
||||
// alternate setting 0 (single-TT) is used.
|
||||
if (!foundEp && !isHub &&
|
||||
iface->bInterfaceClass == CLASS_HUB &&
|
||||
iface->bAlternateSetting == 0) {
|
||||
dev->InterfaceClass = iface->bInterfaceClass;
|
||||
dev->InterfaceSubClass = iface->bInterfaceSubClass;
|
||||
dev->InterfaceProtocol = iface->bInterfaceProtocol;
|
||||
dev->InterfaceNumber = iface->bInterfaceNumber;
|
||||
currentHub = true;
|
||||
isHub = true;
|
||||
}
|
||||
|
||||
if (!foundEp &&
|
||||
iface->bInterfaceClass == CLASS_HID &&
|
||||
iface->bInterfaceSubClass == SUBCLASS_BOOT) {
|
||||
@@ -408,8 +476,8 @@ namespace Drivers::USB::UsbDevice {
|
||||
uint8_t xferType = ep->bmAttributes & EP_XFER_TYPE_MASK;
|
||||
bool isIn = (ep->bEndpointAddress & EP_DIR_IN) != 0;
|
||||
|
||||
// HID interrupt IN endpoint
|
||||
if (foundHid && !foundEp && isIn && xferType == EP_XFER_INTERRUPT) {
|
||||
// HID or hub interrupt IN endpoint
|
||||
if ((foundHid || currentHub) && !foundEp && isIn && xferType == EP_XFER_INTERRUPT) {
|
||||
dev->InterruptEpNum = ep->bEndpointAddress & 0x0F;
|
||||
dev->InterruptMaxPacket = ep->wMaxPacketSize & 0x7FF;
|
||||
dev->InterruptInterval = ep->bInterval;
|
||||
@@ -455,7 +523,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
// The xHCI slot model currently stores one interface; a future model
|
||||
// can retain every alternate/interface without changing the userspace
|
||||
// claim ABI, which already names the interface number explicitly.
|
||||
bool knownInterface = foundBt || foundMsc ||
|
||||
bool knownInterface = foundBt || foundMsc || isHub ||
|
||||
dev->InterfaceClass == CLASS_HID;
|
||||
if (!knownInterface) {
|
||||
bool inFirstInterface = false;
|
||||
@@ -502,15 +570,23 @@ namespace Drivers::USB::UsbDevice {
|
||||
cfgHdr.bConfigurationValue, 0, 0, nullptr, false);
|
||||
if (cc != Xhci::CC_SUCCESS) {
|
||||
KernelLogStream(ERROR, "USB") << "SET_CONFIGURATION failed, cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 8b: Hub descriptor (the slot context must describe the hub)
|
||||
// -----------------------------------------------------------------
|
||||
if (isHub && (!foundEp || !Hub::ReadDescriptor(slotId, speed, hubInfo))) {
|
||||
KernelLogStream(WARNING, "USB") << "Slot " << (uint64_t)slotId
|
||||
<< ": hub left unconfigured";
|
||||
isHub = false;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 9: Configure Endpoints
|
||||
// -----------------------------------------------------------------
|
||||
if (foundEp || foundBulkIn || foundBulkOut) {
|
||||
auto* inputCtx2 = (Xhci::InputContext*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* inputCtx2 = (Xhci::InputContext*)Xhci::AllocateDmaPage();
|
||||
|
||||
// Start with slot context
|
||||
inputCtx2->ICC.AddFlags = (1 << 0);
|
||||
@@ -528,7 +604,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
if (dci > maxDci) maxDci = dci;
|
||||
|
||||
// Allocate interrupt transfer ring
|
||||
auto* intRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* intRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
|
||||
dev->InterruptRing = intRing;
|
||||
dev->InterruptRingPhys = Memory::SubHHDM(intRing);
|
||||
dev->InterruptRingEnqueue = 0;
|
||||
@@ -542,6 +618,10 @@ namespace Drivers::USB::UsbDevice {
|
||||
|
||||
auto& epCtx = inputCtx2->EP[dci - 1];
|
||||
uint32_t xhciInterval = ConvertInterval(speed, dev->InterruptInterval);
|
||||
// Split transactions through a hub's TT: poll at least every
|
||||
// 8 ms. Cheap, and conservative on the VL805, which Linux
|
||||
// gives a quirk for periodic endpoints behind a TT.
|
||||
if (dev->TtHubSlot != 0 && xhciInterval > 6) xhciInterval = 6;
|
||||
epCtx.Field0 = (xhciInterval << 16);
|
||||
epCtx.Field1 = (3 << 1)
|
||||
| (Xhci::EP_TYPE_INTERRUPT_IN << 3)
|
||||
@@ -556,7 +636,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
inputCtx2->ICC.AddFlags |= (1 << dci);
|
||||
if (dci > maxDci) maxDci = dci;
|
||||
|
||||
auto* bulkInRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* bulkInRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
|
||||
dev->BulkInRing = bulkInRing;
|
||||
dev->BulkInRingPhys = Memory::SubHHDM(bulkInRing);
|
||||
dev->BulkInRingEnqueue = 0;
|
||||
@@ -583,7 +663,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
inputCtx2->ICC.AddFlags |= (1 << dci);
|
||||
if (dci > maxDci) maxDci = dci;
|
||||
|
||||
auto* bulkOutRing = (Xhci::TRB*)Memory::g_pfa->AllocateZeroed();
|
||||
auto* bulkOutRing = (Xhci::TRB*)Xhci::AllocateDmaPage();
|
||||
dev->BulkOutRing = bulkOutRing;
|
||||
dev->BulkOutRingPhys = Memory::SubHHDM(bulkOutRing);
|
||||
dev->BulkOutRingEnqueue = 0;
|
||||
@@ -608,6 +688,19 @@ namespace Drivers::USB::UsbDevice {
|
||||
inputCtx2->Slot.Field0 = (inputCtx2->Slot.Field0 & ~(0x1Fu << 27))
|
||||
| (maxDci << 27);
|
||||
|
||||
// A hub: Hub flag, Number of Ports and, for a high-speed hub, its
|
||||
// TT Think Time (xHCI 6.2.2). The controller needs them to
|
||||
// schedule split transactions for what sits behind it.
|
||||
if (isHub) {
|
||||
inputCtx2->Slot.Field0 |= Xhci::SLOT_HUB;
|
||||
inputCtx2->Slot.Field1 = (inputCtx2->Slot.Field1 & 0x00FFFFFF)
|
||||
| ((uint32_t)hubInfo.Ports << 24);
|
||||
if (speed == Xhci::SPEED_HIGH) {
|
||||
inputCtx2->Slot.Field2 = (inputCtx2->Slot.Field2 & ~(3u << 16))
|
||||
| ((uint32_t)hubInfo.ThinkTime << 16);
|
||||
}
|
||||
}
|
||||
|
||||
// Send Configure Endpoint command
|
||||
Xhci::TRB cfgTrb = {};
|
||||
uint64_t inputCtx2Phys = Memory::SubHHDM(inputCtx2);
|
||||
@@ -620,8 +713,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
if (cc != Xhci::CC_SUCCESS) {
|
||||
KernelLogStream(ERROR, "USB") << "Configure Endpoint failed, slot="
|
||||
<< (uint64_t)slotId << " cc=" << (uint64_t)cc;
|
||||
dev->Active = false;
|
||||
return 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (foundEp) {
|
||||
@@ -713,6 +805,8 @@ namespace Drivers::USB::UsbDevice {
|
||||
KernelLogStream(OK, "USB") << "Slot " << (uint64_t)slotId << ": Bluetooth Adapter"
|
||||
<< " VID:" << base::hex << (uint64_t)dev->VendorId
|
||||
<< " PID:" << (uint64_t)dev->ProductId << base::dec;
|
||||
} else if (isHub) {
|
||||
dev->KernelDriverBound = true;
|
||||
} else if (dev->InterfaceClass == CLASS_MASS_STORAGE &&
|
||||
dev->InterfaceSubClass == SUBCLASS_SCSI &&
|
||||
dev->InterfaceProtocol == PROTOCOL_BULK_ONLY &&
|
||||
@@ -733,7 +827,7 @@ namespace Drivers::USB::UsbDevice {
|
||||
// Publish to userspace only after endpoint configuration and kernel
|
||||
// class-driver binding decisions are complete.
|
||||
dev->Ready = true;
|
||||
return slotId;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -80,6 +80,9 @@ namespace Drivers::USB::UsbDevice {
|
||||
constexpr uint8_t PROTOCOL_KEYBOARD = 0x01;
|
||||
constexpr uint8_t PROTOCOL_MOUSE = 0x02;
|
||||
|
||||
// Hub class
|
||||
constexpr uint8_t CLASS_HUB = 0x09;
|
||||
|
||||
// Wireless Controller class (Bluetooth)
|
||||
constexpr uint8_t CLASS_WIRELESS = 0xE0;
|
||||
constexpr uint8_t SUBCLASS_RF = 0x01;
|
||||
@@ -119,8 +122,24 @@ namespace Drivers::USB::UsbDevice {
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Enumerate a newly connected device on the given port with the given speed.
|
||||
// Where a device is attached: a root hub port, or a port of a hub
|
||||
// (Xhci::UsbDeviceInfo describes the fields).
|
||||
struct DeviceLocation {
|
||||
uint8_t RootPort;
|
||||
uint8_t ParentSlot;
|
||||
uint8_t ParentPort;
|
||||
uint8_t Depth;
|
||||
uint32_t RouteString;
|
||||
uint8_t TtHubSlot;
|
||||
uint8_t TtPort;
|
||||
bool TtMulti;
|
||||
};
|
||||
|
||||
// Enumerate a newly connected device on the given root port with the given speed.
|
||||
// Returns the assigned slot ID, or 0 on failure.
|
||||
uint8_t EnumerateDevice(uint8_t portId, uint32_t speed);
|
||||
|
||||
// Enumerate a device at `location` (a hub port that has been reset).
|
||||
uint8_t EnumerateDevice(const DeviceLocation& location, uint32_t speed);
|
||||
|
||||
};
|
||||
+385
-62
@@ -11,11 +11,14 @@
|
||||
#include "HidKeyboard.hpp"
|
||||
#include "HidMouse.hpp"
|
||||
#include "MassStorage.hpp"
|
||||
#include "Hub.hpp"
|
||||
#include "UserUsb.hpp"
|
||||
#include <Pci/Pci.hpp>
|
||||
#include <Terminal/Terminal.hpp>
|
||||
#include <CppLib/Stream.hpp>
|
||||
#include <Memory/HHDM.hpp>
|
||||
#include <Memory/Dma.hpp>
|
||||
#include <ACPI/GsiRouting.hpp>
|
||||
#include <Memory/Paging.hpp>
|
||||
#include <Memory/PageFrameAllocator.hpp>
|
||||
#include <Libraries/Memory.hpp>
|
||||
@@ -59,6 +62,21 @@ namespace Drivers::USB::Xhci {
|
||||
static std::atomic<bool> g_hotplugProcessing{false};
|
||||
static std::atomic<bool> g_irqDrainPending{false};
|
||||
|
||||
// HID interrupt endpoints that completed with an error. The event drain
|
||||
// does not re-arm them (a halted endpoint takes commands to clear, which
|
||||
// cannot be waited on from there): it records the error, and the
|
||||
// deferred-work pass recovers the endpoint once g_hidRecoverDueMs is
|
||||
// reached. Low/full-speed devices behind a hub's TT (every mouse on a
|
||||
// Raspberry Pi 4) see transaction and split errors often enough that
|
||||
// without this the device goes dead until replugged.
|
||||
// g_hidErrorStreak counts errors since the last good report; it sets the
|
||||
// backoff, and at HID_REENUMERATE_STREAK the device is reset and
|
||||
// enumerated again, as a replug would.
|
||||
static constexpr uint32_t HID_REENUMERATE_STREAK = 6;
|
||||
static std::atomic<uint32_t> g_hidErrorCc[MAX_SLOTS + 1] = {}; // 0 = no recovery due
|
||||
static std::atomic<uint32_t> g_hidErrorStreak[MAX_SLOTS + 1] = {};
|
||||
static std::atomic<uint64_t> g_hidRecoverDueMs[MAX_SLOTS + 1] = {};
|
||||
|
||||
// PollEvents also runs directly from the MSI handler. KernelLogStream uses
|
||||
// a process-context mutex, so printing there can deadlock if the interrupt
|
||||
// preempts a CPU that already owns the terminal lock. Capture the small
|
||||
@@ -243,18 +261,44 @@ namespace Drivers::USB::Xhci {
|
||||
return *(volatile uint32_t*)(g_mmioBase + reg);
|
||||
}
|
||||
|
||||
// TRBs (and the buffers they point at) must be visible before the
|
||||
// controller is told to look.
|
||||
static void WriteDoorbell(uint32_t index, uint32_t value) {
|
||||
Arch::DmaWriteFence();
|
||||
*(volatile uint32_t*)(g_dbBase + index * 4) = value;
|
||||
}
|
||||
|
||||
// Hand a TRB to the controller: everything else in it first, then the
|
||||
// control word that carries its cycle bit. A running ring may fetch the
|
||||
// TRB the moment the cycle bit matches.
|
||||
static inline void PublishTrb(TRB& trb, uint32_t control) {
|
||||
Arch::DmaWriteFence();
|
||||
*(volatile uint32_t*)&trb.Control = control;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// DMA buffer allocation
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Whether the controller's DMA snoops the CPU caches. PCI controllers
|
||||
// are taken to be coherent; a platform controller says so in _CCA.
|
||||
static bool g_dmaCoherent = true;
|
||||
|
||||
uint8_t* AllocateDmaPage(uint64_t& outPhysAddr) {
|
||||
return (uint8_t*)Memory::Dma::AllocatePage(outPhysAddr, g_dmaCoherent);
|
||||
}
|
||||
|
||||
void* AllocateDmaPage() {
|
||||
uint64_t phys;
|
||||
return AllocateDmaPage(phys);
|
||||
}
|
||||
|
||||
void FreeDmaPage(void* page) {
|
||||
Memory::Dma::FreePage(page, g_dmaCoherent);
|
||||
}
|
||||
|
||||
static uint8_t* AllocateDmaBuffer(uint64_t& outPhysAddr) {
|
||||
void* virt = Memory::g_pfa->AllocateZeroed();
|
||||
outPhysAddr = Memory::SubHHDM(virt);
|
||||
return (uint8_t*)virt;
|
||||
return AllocateDmaPage(outPhysAddr);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -419,6 +463,24 @@ namespace Drivers::USB::Xhci {
|
||||
// MSI setup (same pattern as E1000E)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Legacy INTx, for controllers without MSI (or platforms that cannot
|
||||
// receive it): the line the firmware or Pci::AssignInterruptLines put
|
||||
// in the Interrupt Line register. HandleInterrupt suits a level line:
|
||||
// clearing IMAN.IP drops it.
|
||||
static bool SetupIntx(uint8_t bus, uint8_t dev, uint8_t func) {
|
||||
uint8_t line = Pci::LegacyRead8(bus, dev, func, (uint8_t)Pci::PCI_REG_INTERRUPT);
|
||||
if (line == 0 || line >= 24) return false;
|
||||
|
||||
uint16_t pciCmd = Pci::LegacyRead16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND);
|
||||
Pci::LegacyWrite16(bus, dev, func, (uint8_t)Pci::PCI_REG_COMMAND,
|
||||
pciCmd & ~Pci::PCI_CMD_INTX_DISABLE);
|
||||
Arch::RegisterIrqHandler(line, HandleInterrupt);
|
||||
Arch::EnableLegacyIrq(line);
|
||||
|
||||
KernelLogStream(OK, "xHCI") << "Using INTx: platform line " << base::dec << (uint64_t)line;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool SetupMsi(uint8_t bus, uint8_t dev, uint8_t func) {
|
||||
uint8_t cap = Pci::FindCapability(bus, dev, func, Pci::PCI_CAP_MSI);
|
||||
if (cap == 0) {
|
||||
@@ -523,6 +585,8 @@ namespace Drivers::USB::Xhci {
|
||||
if (evtCycle != g_evtRingCCS) {
|
||||
break; // No more events
|
||||
}
|
||||
// The rest of the event was written before its cycle bit.
|
||||
Arch::DmaReadFence();
|
||||
|
||||
uint32_t trbType = (evt.Control & TRB_TYPE_MASK) >> TRB_TYPE_SHIFT;
|
||||
lastType = trbType;
|
||||
@@ -545,8 +609,12 @@ namespace Drivers::USB::Xhci {
|
||||
WriteOp(OP_PORTSC_BASE + (portId - 1) * OP_PORTSC_STRIDE,
|
||||
(portsc & PORTSC_PRESERVE) | PORTSC_CHANGE_BITS);
|
||||
|
||||
// Defer enumeration to ProcessDeferredWork outside interrupt context.
|
||||
if (g_bootScanComplete && portId >= 1 && portId <= g_maxPorts) {
|
||||
// Defer enumeration to ProcessDeferredWork outside interrupt
|
||||
// context. Changes during the boot scan are kept too (that
|
||||
// pass skips ports already enumerated): a device that
|
||||
// connects after the scan looked at its port, slower on
|
||||
// real hardware than in QEMU, would otherwise be lost.
|
||||
if (portId >= 1 && portId <= g_maxPorts) {
|
||||
g_hotplugPending[portId - 1].store(true, std::memory_order_release);
|
||||
g_deferredWorkPending.store(true, std::memory_order_release);
|
||||
}
|
||||
@@ -642,12 +710,17 @@ namespace Drivers::USB::Xhci {
|
||||
uint16_t len = (residual < reqLen)
|
||||
? (uint16_t)(reqLen - residual) : 0;
|
||||
|
||||
if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
|
||||
if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
|
||||
// Re-armed once the change is handled.
|
||||
Hub::OnStatusChange((uint8_t)slotId, g_interruptDataBuf[slotId],
|
||||
len, completionCode);
|
||||
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID) {
|
||||
if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
|
||||
HidKeyboard::ProcessReport(g_interruptDataBuf[slotId], len);
|
||||
} else if (dev.InterfaceProtocol == UsbDevice::PROTOCOL_MOUSE) {
|
||||
HidMouse::ProcessReport(g_interruptDataBuf[slotId], len);
|
||||
}
|
||||
g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
|
||||
// Re-queue for next HID report
|
||||
QueueInterruptTransfer(slotId);
|
||||
} else if (g_transferCallbacks[slotId]) {
|
||||
@@ -672,6 +745,23 @@ namespace Drivers::USB::Xhci {
|
||||
g_diagnosticWorkPending.store(true, std::memory_order_release);
|
||||
}
|
||||
|
||||
if (dev.InterfaceClass == UsbDevice::CLASS_HUB && dev.InterruptEpNum &&
|
||||
epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
|
||||
Hub::OnStatusChange((uint8_t)slotId, nullptr, 0, completionCode);
|
||||
}
|
||||
|
||||
if (dev.InterfaceClass == UsbDevice::CLASS_HID && dev.InterruptEpNum &&
|
||||
epDci == (uint32_t)(dev.InterruptEpNum * 2 + 1)) {
|
||||
// Retry at once, then back off 10 ms doubling.
|
||||
uint32_t streak = g_hidErrorStreak[slotId].fetch_add(
|
||||
1, std::memory_order_relaxed) + 1;
|
||||
uint64_t delay = streak <= 1 ? 0 : 10ull << (streak - 2);
|
||||
if (delay > 1000) delay = 1000;
|
||||
g_hidRecoverDueMs[slotId].store(
|
||||
Timekeeping::GetMilliseconds() + delay, std::memory_order_relaxed);
|
||||
g_hidErrorCc[slotId].store(completionCode, std::memory_order_release);
|
||||
}
|
||||
|
||||
// Notify callback of errors too
|
||||
if (g_transferCallbacks[slotId]) {
|
||||
g_transferCallbacks[slotId](slotId, epDci,
|
||||
@@ -787,7 +877,7 @@ namespace Drivers::USB::Xhci {
|
||||
if (g_cmdRingCCS) {
|
||||
control |= TRB_CYCLE_BIT;
|
||||
}
|
||||
slot.Control = control;
|
||||
PublishTrb(slot, control);
|
||||
|
||||
// Advance enqueue pointer
|
||||
g_cmdRingEnqueue++;
|
||||
@@ -836,10 +926,21 @@ namespace Drivers::USB::Xhci {
|
||||
// ControlTransfer - perform a control transfer on EP0
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
|
||||
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
|
||||
void* data, uint64_t dataPhys, bool dirIn, bool trueNested);
|
||||
|
||||
// Data stages of a non-coherent controller go through this page (taken
|
||||
// under g_controlXferLock). Callers pass small stack buffers, and cache
|
||||
// maintenance cannot make those safe: a cache line the buffer shares with
|
||||
// live stack data that the CPU dirties while the transfer runs is written
|
||||
// back over what the device wrote.
|
||||
static uint8_t* g_ctrlBounce = nullptr;
|
||||
static uint64_t g_ctrlBouncePhys = 0;
|
||||
|
||||
uint32_t ControlTransfer(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
|
||||
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
|
||||
void* data, bool dirIn) {
|
||||
|
||||
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) {
|
||||
return 0xFF;
|
||||
}
|
||||
@@ -853,6 +954,32 @@ namespace Drivers::USB::Xhci {
|
||||
g_pollOwnerCpu.load(std::memory_order_relaxed) == CurrentCpuIndex();
|
||||
if (!trueNested) g_controlXferLock.Acquire();
|
||||
|
||||
bool hasData = data != nullptr && wLength != 0;
|
||||
bool bounce = hasData && !g_dmaCoherent && !trueNested &&
|
||||
g_ctrlBounce != nullptr && wLength <= 0x1000;
|
||||
uint32_t cc;
|
||||
if (bounce) {
|
||||
// Copied in both directions: bytes a short IN transfer leaves
|
||||
// alone keep the caller's contents, as with a direct transfer.
|
||||
memcpy(g_ctrlBounce, data, wLength);
|
||||
cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex,
|
||||
wLength, g_ctrlBounce, g_ctrlBouncePhys, dirIn, false);
|
||||
if (dirIn) memcpy(data, g_ctrlBounce, wLength);
|
||||
} else {
|
||||
// The data stage buffer is the caller's (cacheable) memory.
|
||||
if (hasData) Memory::Dma::SyncForDevice(data, wLength, g_dmaCoherent);
|
||||
cc = ControlTransferLocked(slotId, bmRequestType, bRequest, wValue, wIndex, wLength,
|
||||
data, hasData ? Memory::SubHHDM(data) : 0, dirIn, trueNested);
|
||||
if (hasData && dirIn) Memory::Dma::SyncForCpu(data, wLength, g_dmaCoherent);
|
||||
}
|
||||
|
||||
if (!trueNested) g_controlXferLock.Release();
|
||||
return cc;
|
||||
}
|
||||
|
||||
static uint32_t ControlTransferLocked(uint8_t slotId, uint8_t bmRequestType, uint8_t bRequest,
|
||||
uint16_t wValue, uint16_t wIndex, uint16_t wLength,
|
||||
void* data, uint64_t dataPhys, bool dirIn, bool trueNested) {
|
||||
UsbDeviceInfo& dev = g_devices[slotId];
|
||||
|
||||
// --- Setup Stage TRB ---
|
||||
@@ -870,15 +997,15 @@ namespace Drivers::USB::Xhci {
|
||||
if (dev.EP0RingCCS) {
|
||||
setupControl |= TRB_CYCLE_BIT;
|
||||
}
|
||||
setup.Control = setupControl;
|
||||
// The Setup TRB's cycle bit is published only after the Data and
|
||||
// Status TRBs are complete (below): if EP0 is still running, it
|
||||
// would otherwise run into a half-built TD.
|
||||
|
||||
// Advance EP0 enqueue (handles Link TRB wrap)
|
||||
AdvanceEP0Ring(dev);
|
||||
|
||||
// --- Data Stage TRB (if wLength > 0) ---
|
||||
if (wLength > 0 && data != nullptr) {
|
||||
uint64_t dataPhys = Memory::SubHHDM(data);
|
||||
|
||||
TRB& dataTrb = dev.EP0Ring[dev.EP0RingEnqueue];
|
||||
dataTrb.Parameter0 = (uint32_t)(dataPhys & 0xFFFFFFFF);
|
||||
dataTrb.Parameter1 = (uint32_t)(dataPhys >> 32);
|
||||
@@ -891,7 +1018,7 @@ namespace Drivers::USB::Xhci {
|
||||
if (dev.EP0RingCCS) {
|
||||
dataControl |= TRB_CYCLE_BIT;
|
||||
}
|
||||
dataTrb.Control = dataControl;
|
||||
PublishTrb(dataTrb, dataControl);
|
||||
|
||||
AdvanceEP0Ring(dev);
|
||||
}
|
||||
@@ -912,10 +1039,12 @@ namespace Drivers::USB::Xhci {
|
||||
if (dev.EP0RingCCS) {
|
||||
statusControl |= TRB_CYCLE_BIT;
|
||||
}
|
||||
status.Control = statusControl;
|
||||
PublishTrb(status, statusControl);
|
||||
|
||||
AdvanceEP0Ring(dev);
|
||||
|
||||
PublishTrb(setup, setupControl);
|
||||
|
||||
// Ring doorbell for this slot, target EP0 (DCI 1)
|
||||
g_xferCompleted[slotId] = false;
|
||||
WriteDoorbell(slotId, 1);
|
||||
@@ -939,7 +1068,6 @@ namespace Drivers::USB::Xhci {
|
||||
while (Timekeeping::GetMilliseconds() - xferStart < 2000) {
|
||||
PollEvents();
|
||||
if (g_xferCompleted[slotId]) {
|
||||
g_controlXferLock.Release();
|
||||
return g_xferCompletionCode[slotId];
|
||||
}
|
||||
for (int j = 0; j < 100; j++) {
|
||||
@@ -973,7 +1101,6 @@ namespace Drivers::USB::Xhci {
|
||||
| (1 << 16); // DCI=1 (EP0)
|
||||
SendCommand(deqTrb);
|
||||
|
||||
g_controlXferLock.Release();
|
||||
return 0xFF;
|
||||
}
|
||||
|
||||
@@ -1003,7 +1130,7 @@ namespace Drivers::USB::Xhci {
|
||||
if (dev.InterruptRingCCS) {
|
||||
control |= TRB_CYCLE_BIT;
|
||||
}
|
||||
trb.Control = control;
|
||||
PublishTrb(trb, control);
|
||||
|
||||
// Advance enqueue (handles Link TRB wrap)
|
||||
AdvanceInterruptRing(dev);
|
||||
@@ -1140,7 +1267,7 @@ namespace Drivers::USB::Xhci {
|
||||
if (dev.BulkInRingCCS) {
|
||||
control |= TRB_CYCLE_BIT;
|
||||
}
|
||||
trb.Control = control;
|
||||
PublishTrb(trb, control);
|
||||
|
||||
AdvanceBulkInRing(dev);
|
||||
|
||||
@@ -1267,6 +1394,9 @@ namespace Drivers::USB::Xhci {
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
void QueueBulkOutTransfer(uint8_t slotId, uint8_t* data, uint64_t dataPhys, uint32_t length) {
|
||||
if (data != nullptr && length != 0) {
|
||||
Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
|
||||
}
|
||||
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
|
||||
|
||||
UsbDeviceInfo& dev = g_devices[slotId];
|
||||
@@ -1281,7 +1411,7 @@ namespace Drivers::USB::Xhci {
|
||||
if (dev.BulkOutRingCCS) {
|
||||
control |= TRB_CYCLE_BIT;
|
||||
}
|
||||
trb.Control = control;
|
||||
PublishTrb(trb, control);
|
||||
|
||||
AdvanceBulkOutRing(dev);
|
||||
|
||||
@@ -1294,8 +1424,23 @@ namespace Drivers::USB::Xhci {
|
||||
// BulkTransfer - blocking bulk IN/OUT transfer
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
|
||||
uint32_t length, uint32_t* actualLength);
|
||||
|
||||
uint32_t BulkTransfer(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
|
||||
uint32_t length, uint32_t* actualLength) {
|
||||
if (data != nullptr && length != 0) {
|
||||
Memory::Dma::SyncForDevice(data, length, g_dmaCoherent);
|
||||
}
|
||||
uint32_t cc = BulkTransferUnsynced(slotId, dirIn, data, dataPhys, length, actualLength);
|
||||
if (data != nullptr && length != 0 && dirIn) {
|
||||
Memory::Dma::SyncForCpu(data, length, g_dmaCoherent);
|
||||
}
|
||||
return cc;
|
||||
}
|
||||
|
||||
static uint32_t BulkTransferUnsynced(uint8_t slotId, bool dirIn, void* data, uint64_t dataPhys,
|
||||
uint32_t length, uint32_t* actualLength) {
|
||||
if (actualLength) *actualLength = 0;
|
||||
if (length == 0) return CC_SUCCESS;
|
||||
if (length > 0x10000) return 0xFF;
|
||||
@@ -1318,7 +1463,7 @@ namespace Drivers::USB::Xhci {
|
||||
|
||||
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC | TRB_ISP;
|
||||
if (dev.BulkInRingCCS) control |= TRB_CYCLE_BIT;
|
||||
trb.Control = control;
|
||||
PublishTrb(trb, control);
|
||||
|
||||
AdvanceBulkInRing(dev);
|
||||
target = dev.BulkInEpNum * 2 + 1;
|
||||
@@ -1332,7 +1477,7 @@ namespace Drivers::USB::Xhci {
|
||||
|
||||
uint32_t control = (TRB_NORMAL << TRB_TYPE_SHIFT) | TRB_IOC;
|
||||
if (dev.BulkOutRingCCS) control |= TRB_CYCLE_BIT;
|
||||
trb.Control = control;
|
||||
PublishTrb(trb, control);
|
||||
|
||||
AdvanceBulkOutRing(dev);
|
||||
target = dev.BulkOutEpNum * 2;
|
||||
@@ -1412,6 +1557,17 @@ namespace Drivers::USB::Xhci {
|
||||
return g_initialized;
|
||||
}
|
||||
|
||||
// A HID endpoint whose error backoff has run out.
|
||||
static bool HidRecoveryDue() {
|
||||
uint64_t now = 0;
|
||||
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
|
||||
if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
|
||||
if (now == 0) now = Timekeeping::GetMilliseconds();
|
||||
if (now >= g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool HasDeferredWork() {
|
||||
if (g_initialized && g_pollMode && EventRingHasWork()) {
|
||||
g_irqDrainPending.store(true, std::memory_order_release);
|
||||
@@ -1419,6 +1575,7 @@ namespace Drivers::USB::Xhci {
|
||||
return g_initialized &&
|
||||
(g_irqDrainPending.load(std::memory_order_acquire) ||
|
||||
g_deferredWorkPending.load(std::memory_order_acquire) ||
|
||||
(g_bootScanComplete && (Hub::HasPendingWork() || HidRecoveryDue())) ||
|
||||
g_diagnosticWorkPending.load(std::memory_order_acquire));
|
||||
}
|
||||
|
||||
@@ -1468,7 +1625,9 @@ namespace Drivers::USB::Xhci {
|
||||
}
|
||||
|
||||
static void UnregisterClassDriver(uint8_t slotId, const UsbDeviceInfo& dev) {
|
||||
if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
|
||||
if (dev.InterfaceClass == UsbDevice::CLASS_HUB) {
|
||||
Hub::UnregisterHub(slotId);
|
||||
} else if (dev.InterfaceClass == UsbDevice::CLASS_MASS_STORAGE) {
|
||||
MassStorage::UnregisterDevice(slotId);
|
||||
} else if (dev.InterfaceClass == UsbDevice::CLASS_HID &&
|
||||
dev.InterfaceProtocol == UsbDevice::PROTOCOL_KEYBOARD) {
|
||||
@@ -1491,6 +1650,105 @@ namespace Drivers::USB::Xhci {
|
||||
}
|
||||
}
|
||||
|
||||
void RemoveDevice(uint8_t slotId) {
|
||||
if (slotId == 0 || slotId > MAX_SLOTS || !g_devices[slotId].Active) return;
|
||||
|
||||
// Children first: a hub's slot outlives the devices addressed through it.
|
||||
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
|
||||
if (s != slotId && g_devices[s].Active && g_devices[s].ParentSlot == slotId) {
|
||||
RemoveDevice(s);
|
||||
}
|
||||
}
|
||||
|
||||
UserUsb::DeviceDisconnected(slotId);
|
||||
UnregisterClassDriver(slotId, g_devices[slotId]);
|
||||
g_devices[slotId].Active = false;
|
||||
g_hidErrorCc[slotId].store(0, std::memory_order_relaxed);
|
||||
g_hidErrorStreak[slotId].store(0, std::memory_order_relaxed);
|
||||
g_transferCallbacks[slotId] = nullptr;
|
||||
DisableSlot(slotId);
|
||||
g_dcbaa[slotId] = 0;
|
||||
g_devices[slotId] = {};
|
||||
KernelLogStream(INFO, "xHCI") << "Slot " << base::dec << (uint64_t)slotId << " deactivated";
|
||||
}
|
||||
|
||||
// Bring a HID device's interrupt endpoint back after a failed transfer.
|
||||
// Reset Endpoint clears a halt (a Context State Error when the error did
|
||||
// not halt it), Set TR Dequeue skips the failed TRB, and CLEAR_FEATURE
|
||||
// (ENDPOINT_HALT) clears a stall on the device and puts its data toggle
|
||||
// back to DATA0 to match the reset host side (USB 2.0 9.4.5). If errors keep coming, the
|
||||
// device is removed and its port reset, as a replug would.
|
||||
static void RecoverHidEndpoint(uint8_t slotId, uint32_t cc) {
|
||||
UsbDeviceInfo& dev = g_devices[slotId];
|
||||
if (!dev.Active || dev.InterfaceClass != UsbDevice::CLASS_HID ||
|
||||
dev.InterruptEpNum == 0 || !dev.InterruptRing) return;
|
||||
|
||||
uint32_t streak = g_hidErrorStreak[slotId].load(std::memory_order_relaxed);
|
||||
if (streak >= HID_REENUMERATE_STREAK) {
|
||||
KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
|
||||
<< ": HID endpoint keeps failing (cc=" << (uint64_t)cc
|
||||
<< "), resetting the device";
|
||||
uint8_t parentSlot = dev.ParentSlot, parentPort = dev.ParentPort;
|
||||
uint8_t rootPort = dev.PortId;
|
||||
RemoveDevice(slotId);
|
||||
if (parentSlot != 0) {
|
||||
Hub::RequestPortCheck(parentSlot, parentPort);
|
||||
} else if (rootPort >= 1 && rootPort <= g_maxPorts) {
|
||||
// Disable the port (PED is write-1-to-clear) so the hot-plug
|
||||
// pass resets it before enumerating.
|
||||
uint32_t reg = OP_PORTSC_BASE + (rootPort - 1) * OP_PORTSC_STRIDE;
|
||||
WriteOp(reg, (ReadOp(reg) & PORTSC_PRESERVE) | PORTSC_PED);
|
||||
g_hotplugPending[rootPort - 1].store(true, std::memory_order_release);
|
||||
g_deferredWorkPending.store(true, std::memory_order_release);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (streak <= 1) {
|
||||
KernelLogStream(WARNING, "xHCI") << "Slot " << base::dec << (uint64_t)slotId
|
||||
<< ": HID transfer failed (cc=" << (uint64_t)cc << "), recovering endpoint";
|
||||
}
|
||||
|
||||
uint8_t dci = dev.InterruptEpNum * 2 + 1;
|
||||
|
||||
TRB resetTrb = {};
|
||||
resetTrb.Control = (TRB_RESET_ENDPOINT << TRB_TYPE_SHIFT)
|
||||
| ((uint32_t)slotId << 24)
|
||||
| ((uint32_t)dci << 16);
|
||||
bool wasHalted = SendCommand(resetTrb) == CC_SUCCESS;
|
||||
|
||||
uint64_t newDeq = dev.InterruptRingPhys
|
||||
+ (uint64_t)dev.InterruptRingEnqueue * sizeof(TRB);
|
||||
if (dev.InterruptRingCCS) newDeq |= 1; // DCS bit
|
||||
|
||||
TRB deqTrb = {};
|
||||
deqTrb.Parameter0 = (uint32_t)(newDeq & 0xFFFFFFFF);
|
||||
deqTrb.Parameter1 = (uint32_t)(newDeq >> 32);
|
||||
deqTrb.Control = (TRB_SET_TR_DEQUEUE << TRB_TYPE_SHIFT)
|
||||
| ((uint32_t)slotId << 24)
|
||||
| ((uint32_t)dci << 16);
|
||||
SendCommand(deqTrb);
|
||||
|
||||
// Only after a real reset: the host toggle is untouched otherwise.
|
||||
if (wasHalted) {
|
||||
ControlTransfer(slotId, 0x02 /* host-to-dev, standard, endpoint */,
|
||||
0x01 /* CLEAR_FEATURE */, 0 /* ENDPOINT_HALT */,
|
||||
(uint16_t)(0x80 | dev.InterruptEpNum), 0, nullptr, false);
|
||||
}
|
||||
|
||||
if (dev.Active) QueueInterruptTransfer(slotId);
|
||||
}
|
||||
|
||||
static void ProcessHidRecovery() {
|
||||
uint64_t now = Timekeeping::GetMilliseconds();
|
||||
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
|
||||
if (g_hidErrorCc[s].load(std::memory_order_acquire) == 0) continue;
|
||||
if (now < g_hidRecoverDueMs[s].load(std::memory_order_relaxed)) continue;
|
||||
uint32_t cc = g_hidErrorCc[s].exchange(0, std::memory_order_acq_rel);
|
||||
if (cc != 0) RecoverHidEndpoint(s, cc);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// ProcessDeferredWork - handle hot-plug outside interrupt context.
|
||||
// Called from the BSP idle path so class-driver probing can wait on timers.
|
||||
@@ -1508,7 +1766,8 @@ namespace Drivers::USB::Xhci {
|
||||
|
||||
bool haveDiagnostics = g_diagnosticWorkPending.load(std::memory_order_acquire);
|
||||
bool haveHotplug = g_bootScanComplete &&
|
||||
g_deferredWorkPending.load(std::memory_order_acquire);
|
||||
(g_deferredWorkPending.load(std::memory_order_acquire) || Hub::HasPendingWork() ||
|
||||
HidRecoveryDue());
|
||||
if (!haveDiagnostics && !haveHotplug) return;
|
||||
|
||||
// CAS claim: any idle core may call this, but only one runs at a time.
|
||||
@@ -1525,6 +1784,9 @@ namespace Drivers::USB::Xhci {
|
||||
return;
|
||||
}
|
||||
|
||||
// First: a device it gives up on is queued for the hot-plug pass below.
|
||||
ProcessHidRecovery();
|
||||
|
||||
g_deferredWorkPending.store(false, std::memory_order_release);
|
||||
|
||||
for (uint32_t port = 0; port < g_maxPorts; port++) {
|
||||
@@ -1585,25 +1847,23 @@ namespace Drivers::USB::Xhci {
|
||||
UsbDevice::EnumerateDevice(port + 1, speed);
|
||||
}
|
||||
} else {
|
||||
// Device disconnected — deactivate its slot
|
||||
// Device disconnected: deactivate its slot (and, for a hub,
|
||||
// the slots of everything behind it)
|
||||
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
|
||||
if (g_devices[s].Active && g_devices[s].PortId == port + 1) {
|
||||
UserUsb::DeviceDisconnected(s);
|
||||
UnregisterClassDriver(s, g_devices[s]);
|
||||
g_devices[s].Active = false;
|
||||
g_transferCallbacks[s] = nullptr;
|
||||
DisableSlot(s);
|
||||
g_dcbaa[s] = 0;
|
||||
g_devices[s] = {};
|
||||
KernelLogStream(INFO, "xHCI") << "Hot-unplug: slot "
|
||||
<< base::dec << (uint64_t)s << " (port "
|
||||
<< (uint64_t)(port + 1) << ") deactivated";
|
||||
if (g_devices[s].Active && g_devices[s].PortId == port + 1 &&
|
||||
g_devices[s].ParentSlot == 0) {
|
||||
KernelLogStream(INFO, "xHCI") << "Hot-unplug: port "
|
||||
<< base::dec << (uint64_t)(port + 1);
|
||||
RemoveDevice(s);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Port changes behind hubs.
|
||||
Hub::ProcessPendingWork();
|
||||
|
||||
g_hotplugProcessing.store(false, std::memory_order_release);
|
||||
}
|
||||
|
||||
@@ -1611,32 +1871,22 @@ namespace Drivers::USB::Xhci {
|
||||
// Probe (called by PCI driver matching framework)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
bool Probe(const Pci::PciDevice& dev) {
|
||||
if (g_initialized) return false;
|
||||
|
||||
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
|
||||
<< base::hex << (uint64_t)dev.Bus << ":"
|
||||
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
|
||||
|
||||
// Read BAR0 and map MMIO region
|
||||
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
|
||||
|
||||
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
|
||||
|
||||
constexpr uint64_t MmioSize = 0x10000;
|
||||
for (uint64_t offset = 0; offset < MmioSize; offset += 0x1000) {
|
||||
// Map the register block, reset the controller and set up its
|
||||
// context and command/event rings. Interrupt delivery is the caller's.
|
||||
static void InitializeController(uint64_t mmioPhys, uint64_t mmioSize) {
|
||||
for (uint64_t offset = 0; offset < mmioSize; offset += 0x1000) {
|
||||
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
|
||||
}
|
||||
|
||||
g_mmioBase = (volatile uint8_t*)Memory::HHDM(mmioPhys);
|
||||
|
||||
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
|
||||
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
|
||||
// Parse capability registers. CAPLENGTH and HCIVERSION share the
|
||||
// first dword; read it whole, since narrower accesses are not safe
|
||||
// behind every PCIe bridge (Linux does the same).
|
||||
uint32_t capBase = ReadCap(CAP_CAPLENGTH);
|
||||
g_capLength = (uint8_t)(capBase & 0xFF);
|
||||
|
||||
// Parse capability registers
|
||||
g_capLength = *(volatile uint8_t*)(g_mmioBase + CAP_CAPLENGTH);
|
||||
|
||||
uint32_t hciVersion = *(volatile uint16_t*)(g_mmioBase + CAP_HCIVERSION);
|
||||
uint32_t hciVersion = capBase >> 16;
|
||||
KernelLogStream(INFO, "xHCI") << "Version: " << base::hex << (uint64_t)hciVersion
|
||||
<< ", CapLength: " << (uint64_t)g_capLength;
|
||||
|
||||
@@ -1656,6 +1906,16 @@ namespace Drivers::USB::Xhci {
|
||||
g_rtBase = g_mmioBase + rtsOff;
|
||||
g_dbBase = g_mmioBase + dbOff;
|
||||
|
||||
// A firmware-described register block may be smaller than what the
|
||||
// capability registers point into (port registers, interrupter 0,
|
||||
// the doorbells): map those too.
|
||||
uint64_t needed = g_capLength + OP_PORTSC_BASE + (uint64_t)((hcsParams1 >> 24) & 0xFF) * OP_PORTSC_STRIDE;
|
||||
if (rtsOff + 0x40 > needed) needed = rtsOff + 0x40;
|
||||
if (dbOff + 4 * ((hcsParams1 & 0xFF) + 1) > needed) needed = dbOff + 4 * ((hcsParams1 & 0xFF) + 1);
|
||||
for (uint64_t offset = mmioSize; offset < needed; offset += 0x1000) {
|
||||
Memory::VMM::g_paging->MapMMIO(mmioPhys + offset, Memory::HHDM(mmioPhys + offset));
|
||||
}
|
||||
|
||||
KernelLogStream(INFO, "xHCI") << "MaxSlots: " << base::dec << (uint64_t)g_maxSlots
|
||||
<< ", MaxPorts: " << (uint64_t)g_maxPorts
|
||||
<< ", ScratchpadBufs: " << (uint64_t)maxScratchpadBufs;
|
||||
@@ -1701,6 +1961,8 @@ namespace Drivers::USB::Xhci {
|
||||
// Program CONFIG
|
||||
WriteOp(OP_CONFIG, g_maxSlots);
|
||||
|
||||
if (!g_dmaCoherent) g_ctrlBounce = AllocateDmaBuffer(g_ctrlBouncePhys);
|
||||
|
||||
// Allocate DCBAA
|
||||
g_dcbaa = (uint64_t*)AllocateDmaBuffer(g_dcbaaPhys);
|
||||
WriteOp(OP_DCBAAP, (uint32_t)(g_dcbaaPhys & 0xFFFFFFFF));
|
||||
@@ -1748,13 +2010,10 @@ namespace Drivers::USB::Xhci {
|
||||
g_evtRingCCS = true;
|
||||
g_evtRingDequeue = 0;
|
||||
KernelLogStream(OK, "xHCI") << "Event ring at phys " << base::hex << g_evtRingPhys;
|
||||
}
|
||||
|
||||
// MSI setup
|
||||
if (!SetupMsi(dev.Bus, dev.Device, dev.Function)) {
|
||||
KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
|
||||
g_pollMode = true;
|
||||
}
|
||||
|
||||
// Enable the interrupter, run the controller, power and enumerate ports.
|
||||
static bool StartController() {
|
||||
// Enable interrupter 0
|
||||
WriteRt(IR0_IMAN, IMAN_IE);
|
||||
WriteRt(IR0_IMOD, IMOD_INTERVAL_100US);
|
||||
@@ -1822,11 +2081,75 @@ namespace Drivers::USB::Xhci {
|
||||
UsbDevice::EnumerateDevice(port + 1, speed);
|
||||
}
|
||||
|
||||
// Connected ports the scan found nothing on (still training, or the
|
||||
// enumeration failed) get another look from the hot-plug path.
|
||||
for (uint32_t port = 0; port < g_maxPorts; port++) {
|
||||
if (!(ReadOp(OP_PORTSC_BASE + port * OP_PORTSC_STRIDE) & PORTSC_CCS)) continue;
|
||||
bool active = false;
|
||||
for (uint8_t s = 1; s <= MAX_SLOTS; s++) {
|
||||
if (g_devices[s].Active && g_devices[s].PortId == port + 1) active = true;
|
||||
}
|
||||
if (!active) {
|
||||
g_hotplugPending[port].store(true, std::memory_order_release);
|
||||
g_deferredWorkPending.store(true, std::memory_order_release);
|
||||
}
|
||||
}
|
||||
|
||||
g_bootScanComplete = true;
|
||||
KernelLogStream(OK, "xHCI") << "Initialization complete";
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
|
||||
bool activeLow, bool dmaCoherent) {
|
||||
if (g_initialized) return false;
|
||||
|
||||
g_dmaCoherent = dmaCoherent;
|
||||
InitializeController(mmioPhys, mmioSize < 0x1000 ? 0x1000 : (mmioSize + 0xFFF) & ~0xFFFULL);
|
||||
|
||||
uint8_t line = gsi != 0 ? Hal::GsiRouting::LineForGsi(gsi, levelTriggered, activeLow) : 0;
|
||||
if (line != 0) {
|
||||
Arch::RegisterIrqHandler(line, HandleInterrupt);
|
||||
Arch::EnableLegacyIrq(line);
|
||||
KernelLogStream(OK, "xHCI") << "Interrupt: GSI " << base::dec << (uint64_t)gsi
|
||||
<< " on platform line " << (uint64_t)line;
|
||||
} else {
|
||||
KernelLogStream(WARNING, "xHCI") << "No interrupt, using poll mode";
|
||||
}
|
||||
// Watch the event ring from the timer as well. The interrupt is a
|
||||
// PCI INTx routed by firmware through a bridge the kernel does not
|
||||
// drive; if it never arrives, input still works (a ring read per
|
||||
// tick costs next to nothing).
|
||||
g_pollMode = true;
|
||||
|
||||
return StartController();
|
||||
}
|
||||
|
||||
bool Probe(const Pci::PciDevice& dev) {
|
||||
if (g_initialized) return false;
|
||||
|
||||
KernelLogStream(OK, "xHCI") << "Found controller at PCI "
|
||||
<< base::hex << (uint64_t)dev.Bus << ":"
|
||||
<< (uint64_t)dev.Device << "." << (uint64_t)dev.Function;
|
||||
|
||||
uint64_t mmioPhys = Pci::ReadBar0(dev.Bus, dev.Device, dev.Function);
|
||||
|
||||
KernelLogStream(INFO, "xHCI") << "BAR0 physical: " << base::hex << mmioPhys;
|
||||
|
||||
Pci::EnableBusMaster(dev.Bus, dev.Device, dev.Function);
|
||||
KernelLogStream(OK, "xHCI") << "Bus mastering enabled";
|
||||
|
||||
InitializeController(mmioPhys, 0x10000);
|
||||
|
||||
// MSI setup
|
||||
if (!SetupMsi(dev.Bus, dev.Device, dev.Function) && !SetupIntx(dev.Bus, dev.Device, dev.Function)) {
|
||||
KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
|
||||
g_pollMode = true;
|
||||
}
|
||||
|
||||
return StartController();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Initialize (standalone fallback path)
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -2064,8 +2387,8 @@ namespace Drivers::USB::Xhci {
|
||||
// -----------------------------------------------------------------
|
||||
// Step 12: MSI setup
|
||||
// -----------------------------------------------------------------
|
||||
if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function)) {
|
||||
KernelLogStream(WARNING, "xHCI") << "MSI not available, using poll mode";
|
||||
if (!SetupMsi(foundDev->Bus, foundDev->Device, foundDev->Function) && !SetupIntx(foundDev->Bus, foundDev->Device, foundDev->Function)) {
|
||||
KernelLogStream(WARNING, "xHCI") << "No MSI or INTx, using poll mode";
|
||||
g_pollMode = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -177,6 +177,10 @@ namespace Drivers::USB::Xhci {
|
||||
// Device Context structures (xHCI spec section 6.2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Slot Context Field0 flags
|
||||
constexpr uint32_t SLOT_MTT = (1u << 25); // Multi-TT
|
||||
constexpr uint32_t SLOT_HUB = (1u << 26); // the device is a hub
|
||||
|
||||
struct SlotContext {
|
||||
uint32_t Field0; // Route String, Speed, MTT, Hub, Context Entries
|
||||
uint32_t Field1; // Max Exit Latency, Root Hub Port Number, Num Ports
|
||||
@@ -232,8 +236,21 @@ namespace Drivers::USB::Xhci {
|
||||
struct UsbDeviceInfo {
|
||||
bool Active;
|
||||
bool Ready; // descriptors/endpoints and binding are complete
|
||||
uint8_t PortId;
|
||||
uint8_t PortId; // root hub port the device's branch hangs off
|
||||
uint32_t Speed;
|
||||
|
||||
// Topology (xHCI 4.3, 8.9). A device behind a hub records the hub's
|
||||
// slot and port, its route string (one 4-bit port number per hub
|
||||
// tier, tier 1 in bits 3:0) and, for a low/full-speed device behind a
|
||||
// high-speed hub, the hub and port whose transaction translator
|
||||
// carries its traffic.
|
||||
uint8_t ParentSlot; // 0 = attached to a root port
|
||||
uint8_t ParentPort; // port number on the parent hub
|
||||
uint8_t Depth; // hubs between the device and the root port
|
||||
uint32_t RouteString;
|
||||
uint8_t TtHubSlot; // 0 = no transaction translator
|
||||
uint8_t TtPort;
|
||||
bool TtMulti;
|
||||
uint16_t VendorId;
|
||||
uint16_t ProductId;
|
||||
uint8_t InterfaceClass;
|
||||
@@ -295,6 +312,19 @@ namespace Drivers::USB::Xhci {
|
||||
|
||||
void Initialize();
|
||||
bool Probe(const Pci::PciDevice& dev);
|
||||
|
||||
// A controller described by ACPI rather than found on PCI (PNP0D10,
|
||||
// e.g. the Raspberry Pi 4's VL805 behind firmware-configured PCIe):
|
||||
// its register block, its interrupt (an ACPI GSI, 0 = poll) and
|
||||
// whether its DMA snoops the CPU caches (_CCA).
|
||||
bool ProbePlatform(uint64_t mmioPhys, uint64_t mmioSize, uint32_t gsi, bool levelTriggered,
|
||||
bool activeLow, bool dmaCoherent);
|
||||
|
||||
// A zeroed page for rings, contexts and buffers the controller accesses,
|
||||
// mapped to suit the controller's DMA coherency (Memory/Dma.hpp).
|
||||
uint8_t* AllocateDmaPage(uint64_t& outPhysAddr);
|
||||
void* AllocateDmaPage();
|
||||
void FreeDmaPage(void* page);
|
||||
bool IsInitialized();
|
||||
bool HasDeferredWork();
|
||||
|
||||
@@ -374,6 +404,10 @@ namespace Drivers::USB::Xhci {
|
||||
// Access device info
|
||||
UsbDeviceInfo* GetDevice(uint8_t slotId);
|
||||
|
||||
// Tear down a device that went away, and everything behind it if it is a
|
||||
// hub: userspace claims, the class driver, the slot. Process context.
|
||||
void RemoveDevice(uint8_t slotId);
|
||||
|
||||
// Poll the event ring from process/idle context. The hard interrupt only
|
||||
// acknowledges and masks the source, then queues deferred draining.
|
||||
void PollEvents();
|
||||
|
||||
Reference in new issue
Block a user