/*
    * Hci.cpp
    * Bluetooth HCI transport over USB
    * Copyright (c) 2026 Daniel Hammer
*/

#include "Hci.hpp"
#include "L2cap.hpp"
#include <atomic>
#include <Fs/Vfs.hpp>
#include <Drivers/USB/Xhci.hpp>
#include <Drivers/USB/UsbDevice.hpp>
#include <Terminal/Terminal.hpp>
#include <CppLib/Stream.hpp>
#include <CppLib/Spinlock.hpp>
#include <Memory/HHDM.hpp>
#include <Memory/PageFrameAllocator.hpp>
#include <Libraries/Memory.hpp>
#include <Timekeeping/ApicTimer.hpp>

using namespace Kt;

namespace Drivers::USB::Bluetooth::Hci {

    // =========================================================================
    // State
    // =========================================================================

    static uint8_t g_slotId = 0;
    static bool g_initialized = false;

    // Event receive buffer (filled by xHCI interrupt IN callback)
    static uint8_t g_eventBuf[256] = {};
    static volatile uint32_t g_eventLen = 0;
    static volatile bool g_eventReady = false;

    // Firmware-phase diagnostics.  g_fwTrace turns on a bounded per-completion
    // trace of the interrupt IN pipe (SetFwTrace, driven by the bring-up);
    // g_intInCompletions counts interrupt-IN completions since power-up so
    // wait-loop timeouts can report whether the pipe was delivering at all.
    //
    // The trace is recorded into a lock-free ring and printed LATER by
    // DumpFwTrace() from process context.  It must NEVER KernelLogStream from
    // TransferCallback: that runs nested under PollEvents in xHCI MSI (IRQ)
    // context, and the terminal lock is a non-IRQ-disabling Mutex -- an IRQ
    // logger interrupting a same-core holder spins forever (observed: hard
    // boot freeze right after "Intel version raw", 2026-07-06).
    static std::atomic<bool> g_fwTrace{false};
    static std::atomic<uint32_t> g_fwTraceCount{0};
    static std::atomic<uint32_t> g_intInCompletions{0};

    struct FwTraceRec {
        uint8_t  cc;        // xHCI completion code
        uint8_t  len;       // bytes delivered (0 = error/ZLP completion)
        uint8_t  b0;        // first event byte (0xEE = no data)
        uint8_t  ncmd;      // byte 2 (HCI flow-control window in a CC)
        uint16_t b3b4;      // bytes 3..4 (opcode in a Command Complete)
        uint8_t  overwrote; // mailbox still held an unconsumed event
    };
    static constexpr uint32_t FW_TRACE_CAP = 48;
    static FwTraceRec g_fwTraceRing[FW_TRACE_CAP] = {};
    static uint32_t g_fwTraceDumped = 0;   // process-context cursor

    void SetFwTrace(bool on) {
        g_fwTrace.store(on, std::memory_order_relaxed);
        if (on) {
            g_fwTraceCount.store(0, std::memory_order_relaxed);
            g_fwTraceDumped = 0;
        }
    }

    // Print any not-yet-printed trace records.  Process context ONLY.
    void DumpFwTrace() {
        uint32_t n = g_fwTraceCount.load(std::memory_order_acquire);
        if (n > FW_TRACE_CAP) n = FW_TRACE_CAP;
        for (; g_fwTraceDumped < n; g_fwTraceDumped++) {
            const FwTraceRec& r = g_fwTraceRing[g_fwTraceDumped];
            KernelLogStream(INFO, "BT-TRACE") << "int-in cc=" << (uint64_t)r.cc
                << " len=" << (uint64_t)r.len
                << " b0=" << base::hex << (uint64_t)r.b0
                << " b3b4=" << (uint64_t)r.b3b4
                << " ncmd=" << (uint64_t)r.ncmd << base::dec
                << (r.overwrote ? " [mailbox-full]" : "");
        }
    }

    // ACL receive ring buffer.  The bulk-IN callback (nested under PollEvents)
    // only copies an incoming packet into a slot; DrainEvents() processes them
    // at top level.  A ring (not a single buffer) is required because the headset
    // bursts many ACL packets at once -- the single buffer was overwriting and
    // dropping the L2CAP Config Response, leaving our channel half-configured.
    static constexpr int ACL_RX_SLOTS = 32;
    static constexpr int ACL_RX_SLOT_SIZE = 1024;
    static uint8_t g_aclRxRing[ACL_RX_SLOTS][ACL_RX_SLOT_SIZE] = {};
    static volatile uint16_t g_aclRxLens[ACL_RX_SLOTS] = {};
    static volatile uint8_t g_aclRxHead = 0;
    static volatile uint8_t g_aclRxTail = 0;

    // ACL transmit DMA buffers (a ring, not one): SendAcl queues an async bulk
    // OUT transfer, so two sends in quick succession (e.g. our Config Request
    // then a Config Response, both fired while DrainEvents processes a burst)
    // would have the second overwrite the first buffer before it is DMA'd to the
    // wire -- corrupting the first packet.  Rotate buffers to avoid that.
    static constexpr int ACL_TX_SLOTS = 8;
    static uint8_t* g_aclTxRing[ACL_TX_SLOTS] = {};
    static uint64_t g_aclTxRingPhys[ACL_TX_SLOTS] = {};
    static uint8_t g_aclTxSlot = 0;
    // Serializes the whole send (slot grab + packet build + xHCI bulk OUT
    // enqueue).  PumpMedia and DrainEvents have SEPARATE single-instance
    // guards, so one core can be sending a media packet while another sends a
    // signaling reply; without this lock the two race on g_aclTxSlot and on
    // the bulk OUT TRB ring's enqueue/cycle state, garbling packets on air.
    // IRQ-disabling spinlock so the holder also can't be preempted mid-send.
    static kcp::Spinlock g_aclTxLock;

    // HCI command DMA buffer (separate from ACL to avoid conflicts)
    static uint8_t* g_cmdDmaBuf = nullptr;
    static uint64_t g_cmdDmaBufPhys = 0;

    // Connection table
    static ConnectionInfo g_connections[MAX_CONNECTIONS] = {};

    // ACL buffer size (from controller)
    static uint16_t g_aclMaxLen = 0;
    static uint16_t g_aclMaxNum = 0;
    // Controller ACL buffer credits in use: incremented per SendAcl, decremented
    // ONLY by Number-Of-Completed-Packets events.  The bulk OUT USB completion
    // must NOT decrement this -- it only means the packet reached the controller,
    // not that the controller sent it over the air.  (Decrementing on both used
    // to zero the count instantly, defeating A2DP flow control entirely.)
    // Atomic: incremented from syscall context, decremented from the event path
    // on another core.  int32 so a stray double-NOCP clamps instead of wrapping.
    static std::atomic<int32_t> g_aclPendingCount{0};

    // ACL TX flow-control counters: TX submitted and TX completed (bulk OUT
    // completion).  tx - txDone is the number of TX DMA ring slots still owned
    // by the xHCI (see AclTxReady), so the media writer throttles on it.
    static std::atomic<uint32_t> g_aclTxCount{0};
    static std::atomic<uint32_t> g_aclTxDoneCount{0};

    // Inquiry results
    static InquiryDevice g_inquiryResults[MAX_INQUIRY_RESULTS] = {};
    static volatile int g_inquiryResultCount = 0;
    static volatile bool g_inquiryActive = false;

    // Set when the Intel "bootup" vendor event arrives after a firmware boot.
    // Written from the USB transfer callback (ProcessEvent), polled by
    // IntelBootFirmware, hence volatile.
    static volatile bool g_intelBootup = false;

    // Latest Intel "secure send result" vendor event (0xFF sub-opcode 0x06).
    // The bootloader uses this, not Command Complete, to report the outcome of
    // secure-send (0xFC09) firmware download.  Written from ProcessEvent.
    static volatile bool    g_secureResultValid = false;
    static volatile uint8_t g_secureResult = 0;   // 0 = success
    static volatile uint8_t g_secureStatus = 0;   // 0 = success

    // Firmware-download diagnostics.  g_ssBytesSent / g_ssFragsSent accumulate
    // the bytes / 0xFC09 fragments handed to the controller since the last
    // ClearSecureSendResult(), so an async 0xFF/0x06 result or a fragment error
    // can be pinned to an exact upload position.  g_lastControlCC is the xHCI
    // completion code of the most recent SendCommand() control transfer.
    static volatile uint64_t g_ssBytesSent = 0;
    static volatile uint32_t g_ssFragsSent = 0;
    static volatile uint32_t g_lastControlCC = 0;

    // Pending-command queue.  Pairing replies (IO-cap / user-confirm / link-key)
    // are triggered from event handlers running NESTED under PollEvents, where a
    // command can only be fire-and-forget (the reentrancy guard makes a nested
    // wait a no-op).  Fire-and-forget proved unreliable for SSP -- the IO-cap
    // value feeds the authentication confirmation, so a late/garbled reply makes
    // the DHKey check fail (Simple Pairing Complete status 0x05).  Instead the
    // handlers ENQUEUE the reply here; ProcessPendingCommands() sends it later
    // from top-level (process) context with a real, confirmed transfer.
    struct PendingHciCmd { uint16_t opcode; uint8_t len; uint8_t params[16]; };
    static PendingHciCmd g_pending[16] = {};
    static volatile uint8_t g_pendingHead = 0;
    static volatile uint8_t g_pendingTail = 0;

    static void EnqueueHciCmd(uint16_t opcode, const uint8_t* params, uint8_t len) {
        uint8_t next = (uint8_t)((g_pendingHead + 1) & 15);
        if (next == g_pendingTail) return;   // full -> drop (should never happen)
        if (len > 16) len = 16;
        g_pending[g_pendingHead].opcode = opcode;
        g_pending[g_pendingHead].len = len;
        for (uint8_t i = 0; i < len; i++) g_pending[g_pendingHead].params[i] = params[i];
        g_pendingHead = next;
    }

    // =========================================================================
    // Bonded-device link key store
    // =========================================================================
    // Persisted to disk so a pairing survives reboots.  Without it a once-paired
    // device challenges us for the link key on reconnect, we have nothing to
    // answer with, and authentication fails -> the remote drops the link with
    // disconnect reason 0x05.  On EVT_LINK_KEY_NOTIFICATION we cache the new key
    // (RAM, marked dirty); FlushLinkKeys() writes it from process context so the
    // disk I/O never blocks mid-pairing while nested under PollEvents.
    static constexpr int MAX_BONDS = 8;
    static constexpr uint32_t LINK_KEY_MAGIC = 0x314B5442;  // 'BTK1'

    struct StoredLinkKey {
        uint8_t addr[6];
        uint8_t key[16];
        bool    valid;
    };
    static StoredLinkKey g_bonds[MAX_BONDS] = {};
    static bool g_bondsDirty = false;

    static bool AddrEq(const uint8_t* a, const uint8_t* b) {
        for (int i = 0; i < 6; i++) if (a[i] != b[i]) return false;
        return true;
    }

    static int FindBondIndex(const uint8_t* addr) {
        for (int i = 0; i < MAX_BONDS; i++) {
            if (g_bonds[i].valid && AddrEq(g_bonds[i].addr, addr)) return i;
        }
        return -1;
    }

    static void StoreLinkKey(const uint8_t* addr, const uint8_t* key) {
        int idx = FindBondIndex(addr);
        if (idx < 0) {
            for (int i = 0; i < MAX_BONDS; i++) {
                if (!g_bonds[i].valid) { idx = i; break; }
            }
            if (idx < 0) idx = 0;   // table full: recycle the first slot
        }
        memcpy(g_bonds[idx].addr, addr, 6);
        memcpy(g_bonds[idx].key, key, 16);
        g_bonds[idx].valid = true;
        g_bondsDirty = true;        // FlushLinkKeys() persists from safe context
    }

    // On-disk layout: [magic u32][MAX_BONDS x { addr[6], key[16], valid[1] }].
    static constexpr uint64_t LINK_KEY_BLOB_SIZE = 4 + MAX_BONDS * 23;

    void LoadLinkKeys() {
        Fs::Vfs::BackendFile f;
        if (Fs::Vfs::OpenBackendFile("0:/os/btkeys.bin", f) < 0) return;  // none stored yet
        uint64_t size = Fs::Vfs::GetBackendFileSize(f);
        if (size < LINK_KEY_BLOB_SIZE) { Fs::Vfs::CloseBackendFile(f); return; }

        uint8_t blob[LINK_KEY_BLOB_SIZE];
        Fs::Vfs::ReadBackendFile(f, blob, 0, LINK_KEY_BLOB_SIZE);
        Fs::Vfs::CloseBackendFile(f);

        uint32_t magic = (uint32_t)blob[0] | ((uint32_t)blob[1] << 8)
                       | ((uint32_t)blob[2] << 16) | ((uint32_t)blob[3] << 24);
        if (magic != LINK_KEY_MAGIC) return;

        int off = 4, n = 0;
        for (int i = 0; i < MAX_BONDS; i++) {
            memcpy(g_bonds[i].addr, &blob[off], 6); off += 6;
            memcpy(g_bonds[i].key, &blob[off], 16); off += 16;
            g_bonds[i].valid = blob[off++] != 0;
            if (g_bonds[i].valid) n++;
        }
        g_bondsDirty = false;
        KernelLogStream(INFO, "BT-HCI") << "Loaded " << (uint64_t)n << " bonded device key(s)";
    }

    void FlushLinkKeys() {
        if (!g_bondsDirty) return;

        uint8_t blob[LINK_KEY_BLOB_SIZE] = {};
        blob[0] = (uint8_t)(LINK_KEY_MAGIC);
        blob[1] = (uint8_t)(LINK_KEY_MAGIC >> 8);
        blob[2] = (uint8_t)(LINK_KEY_MAGIC >> 16);
        blob[3] = (uint8_t)(LINK_KEY_MAGIC >> 24);
        int off = 4;
        for (int i = 0; i < MAX_BONDS; i++) {
            memcpy(&blob[off], g_bonds[i].addr, 6); off += 6;
            memcpy(&blob[off], g_bonds[i].key, 16); off += 16;
            blob[off++] = g_bonds[i].valid ? 1 : 0;
        }

        Fs::Vfs::BackendFile f;
        if (Fs::Vfs::CreateBackendFile("0:/os/btkeys.bin", f) < 0) {
            KernelLogStream(WARNING, "BT-HCI") << "Could not open link key store for writing";
            return;
        }
        Fs::Vfs::WriteBackendFile(f, blob, 0, LINK_KEY_BLOB_SIZE);
        Fs::Vfs::CloseBackendFile(f);
        g_bondsDirty = false;
        KernelLogStream(OK, "BT-HCI") << "Link key store persisted";
    }

    int ListBonds(BondInfo* buf, int maxCount) {
        if (!buf || maxCount <= 0) return 0;
        int n = 0;
        for (int i = 0; i < MAX_BONDS && n < maxCount; i++) {
            if (g_bonds[i].valid) {
                memcpy(buf[n].Addr, g_bonds[i].addr, 6);
                n++;
            }
        }
        return n;
    }

    bool ForgetBond(const uint8_t* addr) {
        if (!addr) return false;
        int idx = FindBondIndex(addr);
        if (idx < 0) return false;
        g_bonds[idx].valid = false;
        memset(g_bonds[idx].key, 0, 16);
        memset(g_bonds[idx].addr, 0, 6);
        g_bondsDirty = true;
        FlushLinkKeys();   // persist removal now (caller is process context)
        return true;
    }

    // =========================================================================
    // USB transfer callback
    // =========================================================================

    static void TransferCallback(uint8_t slotId, uint8_t epDci,
                                 const uint8_t* data, uint32_t length,
                                 uint32_t completionCode) {
        if (slotId != g_slotId) return;

        auto* dev = Xhci::GetDevice(slotId);
        if (!dev) return;

        uint8_t intDci = dev->InterruptEpNum ? (dev->InterruptEpNum * 2 + 1) : 0;
        uint8_t bulkInDci = dev->BulkInEpNum ? (dev->BulkInEpNum * 2 + 1) : 0;

        if (epDci == intDci) {
            g_intInCompletions.fetch_add(1, std::memory_order_relaxed);

            // Bounded bring-up trace: one RING RECORD per interrupt-IN
            // completion while the firmware phase runs; DumpFwTrace() prints
            // them later from process context.  No logging here -- this runs
            // in xHCI MSI (IRQ) context (see the deadlock note at the ring).
            if (g_fwTrace.load(std::memory_order_relaxed)) {
                uint32_t n = g_fwTraceCount.fetch_add(1, std::memory_order_relaxed);
                if (n < FW_TRACE_CAP) {
                    FwTraceRec& r = g_fwTraceRing[n];
                    r.cc   = (uint8_t)completionCode;
                    r.len  = (uint8_t)(length > 255 ? 255 : length);
                    r.b0   = (data && length >= 1) ? data[0] : 0xEE;
                    r.ncmd = (data && length >= 3) ? data[2] : 0xEE;
                    r.b3b4 = (data && length >= 5)
                        ? (uint16_t)((uint16_t)data[3] | ((uint16_t)data[4] << 8)) : 0xEEEE;
                    r.overwrote = g_eventReady ? 1 : 0;
                }
            }

            if (data && length > 0) {
                // HCI Event received on interrupt IN.
                // Dispatch asynchronous events (inquiry results, connection
                // events, etc.) immediately so they are never lost.  Only
                // buffer Command Complete / Command Status events — those are
                // consumed by WaitCommandComplete / WaitCommandStatus.
                uint8_t evtCode = data[0];

                if (evtCode == EVT_COMMAND_COMPLETE || evtCode == EVT_COMMAND_STATUS) {
                    uint32_t copyLen = length;
                    if (copyLen > sizeof(g_eventBuf)) copyLen = sizeof(g_eventBuf);
                    memcpy(g_eventBuf, data, copyLen);
                    g_eventLen = copyLen;
                    g_eventReady = true;
                } else {
                    // Process immediately (inquiry results, connection events, etc.)
                    ProcessEvent(data, length);
                }
            }

            // ALWAYS re-queue, including error (data == nullptr) and 0-length
            // (ZLP) completions.  Re-arming only on data>0 meant a single such
            // completion silently killed the event pipe for good -- every
            // later command then "timed out" with no xHCI error in sight.
            // Mirrors the bulk-IN rule below ("must keep cycling").
            Xhci::QueueInterruptTransfer(slotId);
        } else if (epDci == bulkInDci) {
            // ACL data received on bulk IN -> copy into the ring; processed by
            // DrainEvents() at top level (do NOT process here, nested).
            // Only enqueue a packet big enough to carry an L2CAP header (ACL
            // header + L2CAP header = 8 bytes).  Smaller completions are ZLP /
            // re-arm artifacts and the firmware-phase bulk-IN runts (4-7 bytes,
            // the old "rx flood") -- ProcessPacket would reject them anyway, and
            // dropping them here keeps the ring + rx stats clean.  We STILL
            // re-arm on every completion below (the bulk IN must keep cycling to
            // absorb the device's ~635 KB cc=4 glitch during firmware download).
            if (data && length >= sizeof(AclHeader) + 4) {
                uint8_t next = (uint8_t)((g_aclRxHead + 1) % ACL_RX_SLOTS);
                if (next != g_aclRxTail) {   // ring not full (else drop: overran)
                    uint32_t copyLen = length;
                    if (copyLen > ACL_RX_SLOT_SIZE) copyLen = ACL_RX_SLOT_SIZE;
                    memcpy(g_aclRxRing[g_aclRxHead], data, copyLen);
                    g_aclRxLens[g_aclRxHead] = (uint16_t)copyLen;
                    g_aclRxHead = next;
                }
            }

            // Re-queue bulk IN transfer (only on a real success/short completion;
            // the error path passes data==nullptr and is handled elsewhere).
            if (data) {
                Xhci::QueueBulkInTransfer(slotId, nullptr, 0, dev->BulkInMaxPacket);
            }
        } else if (epDci == (dev->BulkOutEpNum ? (uint8_t)(dev->BulkOutEpNum * 2) : (uint8_t)0)) {
            // Bulk OUT completion: the packet was DMA'd to the controller and
            // its TX ring slot is free again.  Controller buffer credits
            // (g_aclPendingCount) are NOT released here -- only the
            // Number-Of-Completed-Packets event does that.
            g_aclTxDoneCount.fetch_add(1, std::memory_order_relaxed);
        }
    }

    // =========================================================================
    // Busy wait with event polling
    // =========================================================================

    static void BusyWaitMs(uint64_t ms) {
        uint64_t start = Timekeeping::GetMilliseconds();
        while (Timekeeping::GetMilliseconds() - start < ms) {
            asm volatile("pause" ::: "memory");
        }
    }

    // Poll for events while waiting
    static void PollWait(uint32_t ms) {
        uint64_t start = Timekeeping::GetMilliseconds();
        while (Timekeeping::GetMilliseconds() - start < ms) {
            Xhci::PollEvents();
            for (int j = 0; j < 100; j++) {
                asm volatile("" ::: "memory");
            }
        }
    }

    // =========================================================================
    // Initialize
    // =========================================================================

    void Initialize(uint8_t slotId) {
        g_slotId = slotId;

        // Register our transfer callback
        Xhci::RegisterTransferCallback(slotId, TransferCallback);

        // Allocate DMA buffers for HCI commands and ACL data
        g_cmdDmaBuf = (uint8_t*)Memory::g_pfa->AllocateZeroed();
        g_cmdDmaBufPhys = Memory::SubHHDM(g_cmdDmaBuf);

        for (int i = 0; i < ACL_TX_SLOTS; i++) {
            g_aclTxRing[i] = (uint8_t*)Memory::g_pfa->AllocateZeroed();
            g_aclTxRingPhys[i] = Memory::SubHHDM(g_aclTxRing[i]);
        }

        // NOTE: Do NOT queue interrupt IN or bulk IN transfers here.
        // The BT controller is not yet HCI-initialized and may misbehave.
        // Call StartEventPipe() after HCI Reset and initial setup.

        g_initialized = true;
        KernelLogStream(OK, "BT-HCI") << "HCI transport initialized on slot " << (uint64_t)slotId;
    }

    // Start receiving HCI events and ACL data — call after HCI init sequence.
    // Arms BOTH the interrupt IN (events) and the bulk IN (ACL).  The bulk IN
    // MUST stay armed across the firmware download: on this controller the device
    // glitches a USB transaction error (cc=4) near ~635 KB of the upload, and an
    // armed bulk IN ABSORBS it (a benign cc=4 on the bulk endpoint) so the
    // interrupt-IN event pipe survives and the download completes.  Deferring the
    // bulk-IN arm moves that cc=4 onto the interrupt IN and wedges the download at
    // 635 KB, so the bulk IN must be armed up front.  The harmless firmware-phase
    // bulk-IN runts (4-7 byte completions) are filtered at enqueue in
    // TransferCallback so they never pollute the RX ring.
    void StartEventPipe() {
        if (!g_initialized) return;

        // Queue initial interrupt IN transfer for HCI events
        Xhci::QueueInterruptTransfer(g_slotId);

        // Queue initial bulk IN transfer for ACL data
        auto* dev = Xhci::GetDevice(g_slotId);
        if (dev && dev->BulkInEpNum) {
            Xhci::QueueBulkInTransfer(g_slotId, nullptr, 0, dev->BulkInMaxPacket);
        }

        KernelLogStream(INFO, "BT-HCI") << "Event pipe started (interrupt IN + bulk IN)";
    }

    // =========================================================================
    // SendCommand — via USB control transfer on EP0
    // =========================================================================

    bool SendCommand(uint16_t opcode, const uint8_t* params, uint8_t paramLen) {
        if (!g_initialized || !g_cmdDmaBuf) return false;

        // HCI command packet: opcode (2) + paramLen (1) + params
        // USB-BT spec: HCI commands are sent via control transfer
        // bmRequestType = 0x20 (Host-to-device, Class, Device)
        // bRequest = 0x00
        // wValue = 0, wIndex = 0
        // wLength = sizeof(CommandHeader) + paramLen

        // Use DMA-allocated buffer (not stack) for the command data.
        // xHCI reads from this buffer via DMA for OUT transfers.
        memset(g_cmdDmaBuf, 0, 512);
        g_cmdDmaBuf[0] = (uint8_t)(opcode & 0xFF);
        g_cmdDmaBuf[1] = (uint8_t)(opcode >> 8);
        g_cmdDmaBuf[2] = paramLen;
        if (params && paramLen > 0) {
            memcpy(&g_cmdDmaBuf[3], params, paramLen);
        }

        uint16_t totalLen = 3 + paramLen;

        uint32_t cc = Xhci::ControlTransfer(g_slotId,
            0x20,    // bmRequestType: Host-to-device, Class, Device
            0x00,    // bRequest: 0
            0x0000,  // wValue
            0x0000,  // wIndex
            totalLen,
            g_cmdDmaBuf,
            false);  // dirIn = false (host to device)

        g_lastControlCC = cc;

        if (cc != Xhci::CC_SUCCESS) {
            KernelLogStream(WARNING, "BT-HCI") << "SendCommand failed, opcode="
                << base::hex << (uint64_t)opcode << " cc=" << base::dec << (uint64_t)cc;
            return false;
        }

        return true;
    }

    // =========================================================================
    // WaitCommandComplete
    // =========================================================================

    bool WaitCommandComplete(uint16_t opcode, uint8_t* outParams,
                             uint8_t maxLen, uint32_t timeoutMs) {
        // Nested inside PollEvents (an event handler issued this command): we
        // cannot wait -- a nested PollEvents is a no-op, so the Command Complete
        // is reaped by the active PollEvents after we return.  The command was
        // already submitted (fire-and-forget); report success.
        if (Xhci::InPollContext()) return true;

        uint64_t start = Timekeeping::GetMilliseconds();

        while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
            Xhci::PollEvents();

            if (g_eventReady) {
                g_eventReady = false;

                if (g_eventLen >= 2) {
                    uint8_t evtCode = g_eventBuf[0];
                    uint8_t evtParamLen = g_eventBuf[1];

                    if (evtCode == EVT_COMMAND_COMPLETE && evtParamLen >= 3) {
                        // Command Complete: NumPkts(1) + Opcode(2) + Status(1) + Params
                        uint16_t evtOpcode = (uint16_t)g_eventBuf[3] | ((uint16_t)g_eventBuf[4] << 8);
                        if (evtOpcode == opcode) {
                            if (outParams && maxLen > 0) {
                                // Copy params starting after the status byte
                                uint8_t availLen = (evtParamLen > 4) ? (evtParamLen - 4) : 0;
                                uint8_t copyLen = (availLen < maxLen) ? availLen : maxLen;
                                // Include status byte + return params
                                copyLen = (evtParamLen > 3) ? (evtParamLen - 3) : 0;
                                if (copyLen > maxLen) copyLen = maxLen;
                                memcpy(outParams, &g_eventBuf[5], copyLen);
                            }
                            // Check status
                            uint8_t status = g_eventBuf[5];
                            if (status != 0) {
                                KernelLogStream(WARNING, "BT-HCI") << "Command Complete status="
                                    << (uint64_t)status << " opcode=" << base::hex << (uint64_t)opcode;
                            }
                            return true;
                        }
                    }

                }
            }

            for (int j = 0; j < 100; j++) {
                asm volatile("" ::: "memory");
            }
        }

        KernelLogStream(WARNING, "BT-HCI") << "WaitCommandComplete timeout, opcode="
            << base::hex << (uint64_t)opcode;
        return false;
    }

    // =========================================================================
    // WaitCommandStatus
    // =========================================================================

    bool WaitCommandStatus(uint16_t opcode, uint32_t timeoutMs) {
        // See WaitCommandComplete: cannot wait when nested under PollEvents.
        if (Xhci::InPollContext()) return true;

        uint64_t start = Timekeeping::GetMilliseconds();

        while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
            Xhci::PollEvents();

            if (g_eventReady) {
                g_eventReady = false;

                if (g_eventLen >= 2) {
                    uint8_t evtCode = g_eventBuf[0];
                    uint8_t evtParamLen = g_eventBuf[1];

                    if (evtCode == EVT_COMMAND_STATUS && evtParamLen >= 4) {
                        uint8_t status = g_eventBuf[2];
                        uint16_t evtOpcode = (uint16_t)g_eventBuf[4] | ((uint16_t)g_eventBuf[5] << 8);
                        if (evtOpcode == opcode) {
                            return (status == 0);
                        }
                    }

                }
            }

            for (int j = 0; j < 100; j++) {
                asm volatile("" ::: "memory");
            }
        }

        return false;
    }

    // =========================================================================
    // SendAcl — via USB bulk OUT
    // =========================================================================

    bool SendAcl(uint16_t handle, uint16_t pbFlag, const uint8_t* data, uint16_t len) {
        if (!g_initialized || !g_aclTxRing[0]) return false;
        if (len + sizeof(AclHeader) > 4096) return false;  // Single page DMA buffer

        g_aclTxLock.Acquire();

        // Every slot still owned by the xHCI: reusing one would let this
        // packet's memcpy race the in-flight DMA.  Dropping is recoverable
        // (L2CAP peers retransmit signaling; media just skips a frame),
        // corruption on air is not.
        if (AclTxInFlight() >= (uint32_t)ACL_TX_SLOTS) {
            g_aclTxLock.Release();
            return false;
        }

        // Use the next TX ring slot so a rapid second send can't overwrite this
        // packet before its bulk OUT transfer DMAs it to the wire.
        uint8_t* txBuf = g_aclTxRing[g_aclTxSlot];
        uint64_t txPhys = g_aclTxRingPhys[g_aclTxSlot];
        g_aclTxSlot = (uint8_t)((g_aclTxSlot + 1) % ACL_TX_SLOTS);

        // Build ACL packet in DMA buffer
        auto* hdr = (AclHeader*)txBuf;
        hdr->HandleFlags = (handle & 0x0FFF) | pbFlag;
        hdr->DataLength = len;
        if (data && len > 0) {
            memcpy(txBuf + sizeof(AclHeader), data, len);
        }

        uint32_t totalLen = sizeof(AclHeader) + len;

        g_aclPendingCount.fetch_add(1, std::memory_order_relaxed);
        g_aclTxCount.fetch_add(1, std::memory_order_relaxed);
        Xhci::QueueBulkOutTransfer(g_slotId, txBuf, txPhys, totalLen);

        g_aclTxLock.Release();
        return true;
    }

    uint16_t AclPendingCount() {
        int32_t v = g_aclPendingCount.load(std::memory_order_relaxed);
        return (v > 0) ? (uint16_t)v : 0;
    }
    uint16_t AclMaxPackets()   { return g_aclMaxNum; }

    uint32_t AclTxInFlight() {
        uint32_t tx = g_aclTxCount.load(std::memory_order_relaxed);
        uint32_t done = g_aclTxDoneCount.load(std::memory_order_relaxed);
        return (tx >= done) ? (tx - done) : 0;
    }

    bool AclTxReady() {
        // A send is safe only when BOTH resources have room:
        //  - a controller ACL buffer credit (NOCP-tracked), and
        //  - a TX DMA ring slot not still queued in the xHCI transfer ring.
        // Leave one TX slot of headroom so a signaling reply fired from the
        // event path (AVDTP/AVRCP response) can always go out.
        uint16_t maxOut = g_aclMaxNum ? g_aclMaxNum : 4;
        if (AclPendingCount() >= maxOut) return false;
        if (AclTxInFlight() >= (uint32_t)(ACL_TX_SLOTS - 1)) return false;
        return true;
    }

    void AclResetCredits() {
        // Lost-NOCP recovery: caller observed a prolonged credit stall with the
        // USB side fully drained.  Assume the completions were lost and start
        // over; if the controller is genuinely still full it will NAK bulk OUT,
        // which the TX-slot half of AclTxReady() still backpressures on.
        g_aclPendingCount.store(0, std::memory_order_relaxed);
    }

    // =========================================================================
    // ProcessEvent — handle HCI events
    // =========================================================================

    void ProcessEvent(const uint8_t* data, uint32_t len) {
        if (len < 2) return;

        uint8_t evtCode = data[0];
        uint8_t evtParamLen = data[1];
        const uint8_t* params = data + 2;

        switch (evtCode) {
            case EVT_CONNECTION_COMPLETE: {
                if (evtParamLen >= 11) {
                    uint8_t status = params[0];
                    uint16_t handle = (uint16_t)params[1] | ((uint16_t)params[2] << 8);
                    const uint8_t* bdAddr = &params[3];
                    uint8_t linkType = params[9];

                    KernelLogStream(INFO, "BT-HCI") << "Connection Complete: status="
                        << (uint64_t)status << " handle=" << (uint64_t)handle
                        << " link=" << (uint64_t)linkType;

                    if (status == 0) {
                        // Find empty connection slot
                        for (int i = 0; i < MAX_CONNECTIONS; i++) {
                            if (!g_connections[i].Active) {
                                g_connections[i].Active = true;
                                g_connections[i].Handle = handle;
                                memcpy(g_connections[i].BdAddr, bdAddr, 6);
                                g_connections[i].LinkType = linkType;
                                g_connections[i].Encrypted = false;
                                break;
                            }
                        }

                        // Initialize L2CAP for this connection
                        L2cap::Initialize(handle);
                    }
                }
                break;
            }

            case EVT_DISCONNECTION_COMPLETE: {
                if (evtParamLen >= 4) {
                    uint16_t handle = (uint16_t)params[1] | ((uint16_t)params[2] << 8);
                    uint8_t reason = params[3];

                    KernelLogStream(INFO, "BT-HCI") << "Disconnection: handle="
                        << (uint64_t)handle << " reason=" << (uint64_t)reason;

                    for (int i = 0; i < MAX_CONNECTIONS; i++) {
                        if (g_connections[i].Active && g_connections[i].Handle == handle) {
                            g_connections[i].Active = false;
                            break;
                        }
                    }
                }
                break;
            }

            case EVT_CONNECTION_REQUEST: {
                if (evtParamLen >= 10) {
                    const uint8_t* bdAddr = &params[0];
                    uint8_t linkType = params[9];

                    KernelLogStream(INFO, "BT-HCI") << "Connection Request: link="
                        << (uint64_t)linkType;

                    // Auto-accept ACL connections
                    if (linkType == 0x01) {
                        AcceptConnection(bdAddr, 0x01);  // Role = slave
                    }
                }
                break;
            }

            case EVT_NUM_COMPLETED_PACKETS: {
                if (evtParamLen >= 1) {
                    uint8_t numHandles = params[0];
                    for (int i = 0; i < numHandles && (3 + i * 4) < evtParamLen; i++) {
                        uint16_t completed = (uint16_t)params[3 + i * 4]
                                           | ((uint16_t)params[4 + i * 4] << 8);
                        int32_t v = g_aclPendingCount.fetch_sub(completed,
                                        std::memory_order_relaxed) - completed;
                        if (v < 0) g_aclPendingCount.store(0, std::memory_order_relaxed);
                    }
                }
                break;
            }

            case EVT_IO_CAPABILITY_REQUEST: {
                if (evtParamLen >= 6) {
                    // Reply with NoInputNoOutput for simple pairing
                    uint8_t reply[9] = {};
                    memcpy(reply, &params[0], 6);  // BD_ADDR
                    reply[6] = 0x03;  // IO Capability: NoInputNoOutput
                    reply[7] = 0x00;  // OOB data not present
                    reply[8] = 0x04;  // Auth req: MITM not required, General Bonding
                                      // (0x04, not 0x00 "No Bonding") so a
                                      // persistent link key is created -> the bond
                                      // survives reboots via the link-key store.
                    // Queue for reliable top-level delivery (see EnqueueHciCmd):
                    // the IO-cap value feeds the SSP confirmation, so it must be
                    // sent intact, not fire-and-forget.
                    EnqueueHciCmd(OP_IO_CAPABILITY_REPLY, reply, 9);
                }
                break;
            }

            case EVT_USER_CONFIRM_REQUEST: {
                if (evtParamLen >= 6) {
                    // Auto-confirm (Just Works).  Queue for reliable top-level
                    // delivery -- a late confirm makes pairing fail (spp=0x05).
                    EnqueueHciCmd(OP_USER_CONFIRM_REPLY, &params[0], 6);
                }
                break;
            }

            case EVT_LINK_KEY_REQUEST: {
                if (evtParamLen >= 6) {
                    int idx = FindBondIndex(&params[0]);
                    if (idx >= 0) {
                        // We remember this device: hand back the stored key so
                        // authentication succeeds without re-pairing.
                        uint8_t reply[22];
                        memcpy(reply, &params[0], 6);
                        memcpy(&reply[6], g_bonds[idx].key, 16);
                        EnqueueHciCmd(OP_LINK_KEY_REQ_REPLY, reply, 22);
                    } else {
                        // Unknown device: tell the controller we have no key so
                        // the remote falls back to fresh Secure Simple Pairing
                        // (Just Works) instead of failing auth (reason 0x05).
                        EnqueueHciCmd(OP_LINK_KEY_REQ_NEG_REPLY, &params[0], 6);
                    }
                }
                break;
            }

            case EVT_LINK_KEY_NOTIFICATION: {
                // Pairing produced a new link key: BD_ADDR[6] + key[16] + type[1].
                // Cache it now (fast); FlushLinkKeys() persists it to disk from
                // process context so the disk write never stalls this nested
                // event handler mid-pairing.
                if (evtParamLen >= 22) {
                    StoreLinkKey(&params[0], &params[6]);
                    KernelLogStream(INFO, "BT-HCI") << "Link key notification (new bond cached)";
                }
                break;
            }

            case EVT_AUTH_COMPLETE: {
                // Authentication succeeded -> turn on encryption.  The link must
                // be encrypted before A2DP; without this the headset finishes
                // pairing, waits for encryption that never comes, and drops us
                // (reason 0x05).  Set Connection Encryption = handle(2) + 0x01.
                // Queue for reliable top-level delivery.
                if (evtParamLen >= 3 && params[0] == 0) {
                    uint8_t enc[3] = { params[1], params[2], 0x01 };
                    EnqueueHciCmd(OP_SET_CONN_ENCRYPT, enc, 3);
                }
                break;
            }

            case EVT_INQUIRY_COMPLETE: {
                g_inquiryActive = false;
                KernelLogStream(INFO, "BT-HCI") << "Inquiry complete, "
                    << (uint64_t)g_inquiryResultCount << " device(s) found";
                break;
            }

            case EVT_INQUIRY_RESULT: {
                // Standard inquiry result: NumResp(1) + per-device(14 bytes each)
                if (evtParamLen >= 1) {
                    uint8_t numResp = params[0];
                    for (int i = 0; i < numResp && g_inquiryResultCount < MAX_INQUIRY_RESULTS; i++) {
                        const uint8_t* entry = &params[1 + i * 14];
                        auto& dev = g_inquiryResults[g_inquiryResultCount];
                        memset(&dev, 0, sizeof(dev));
                        memcpy(dev.BdAddr, entry, 6);
                        dev.ClassOfDevice = (uint32_t)entry[9]
                                          | ((uint32_t)entry[10] << 8)
                                          | ((uint32_t)entry[11] << 16);
                        dev.Rssi = -128; // Unknown for standard inquiry
                        g_inquiryResultCount++;
                    }
                }
                break;
            }

            case EVT_INQUIRY_RESULT_RSSI: {
                // Inquiry Result with RSSI: NumResp(1) + per-device(15 bytes each)
                if (evtParamLen >= 1) {
                    uint8_t numResp = params[0];
                    for (int i = 0; i < numResp && g_inquiryResultCount < MAX_INQUIRY_RESULTS; i++) {
                        const uint8_t* entry = &params[1 + i * 15];
                        auto& dev = g_inquiryResults[g_inquiryResultCount];
                        memset(&dev, 0, sizeof(dev));
                        memcpy(dev.BdAddr, entry, 6);
                        dev.ClassOfDevice = (uint32_t)entry[9]
                                          | ((uint32_t)entry[10] << 8)
                                          | ((uint32_t)entry[11] << 16);
                        dev.Rssi = (int8_t)entry[14];
                        g_inquiryResultCount++;
                    }
                }
                break;
            }

            case EVT_EXTENDED_INQUIRY_RESULT: {
                // Extended Inquiry Result: NumResp(1) + BD_ADDR(6) + PSRM(1) + reserved(1)
                //   + CoD(3) + ClockOff(2) + RSSI(1) + EIR(240)
                if (evtParamLen >= 15 && g_inquiryResultCount < MAX_INQUIRY_RESULTS) {
                    auto& dev = g_inquiryResults[g_inquiryResultCount];
                    memset(&dev, 0, sizeof(dev));
                    memcpy(dev.BdAddr, &params[1], 6);
                    dev.ClassOfDevice = (uint32_t)params[9]
                                      | ((uint32_t)params[10] << 8)
                                      | ((uint32_t)params[11] << 16);
                    dev.Rssi = (int8_t)params[14];

                    // Parse EIR data for device name
                    const uint8_t* eir = &params[15];
                    int eirLen = evtParamLen - 15;
                    int pos = 0;
                    while (pos < eirLen && pos < 240) {
                        uint8_t len = eir[pos];
                        if (len == 0) break;
                        if (pos + 1 + len > eirLen) break;
                        uint8_t type = eir[pos + 1];
                        // Type 0x08 = Shortened Local Name, 0x09 = Complete Local Name
                        if (type == 0x08 || type == 0x09) {
                            int nameLen = len - 1;
                            if (nameLen > 63) nameLen = 63;
                            memcpy(dev.Name, &eir[pos + 2], nameLen);
                            dev.Name[nameLen] = '\0';
                        }
                        pos += 1 + len;
                    }

                    g_inquiryResultCount++;
                }
                break;
            }

            case EVT_ENCRYPT_CHANGE: {
                if (evtParamLen >= 4) {
                    uint16_t handle = (uint16_t)params[1] | ((uint16_t)params[2] << 8);
                    uint8_t encryption = params[3];

                    for (int i = 0; i < MAX_CONNECTIONS; i++) {
                        if (g_connections[i].Active && g_connections[i].Handle == handle) {
                            g_connections[i].Encrypted = (encryption != 0);
                            break;
                        }
                    }
                }
                break;
            }

            case EVT_VENDOR_SPECIFIC: {
                // Intel vendor events carry a sub-opcode in the first byte.
                //   0x02 = "bootup" (operational firmware booted after 0xFC01)
                //   0x06 = "secure send result": result(1) opcode(2) status(1)
                uint8_t sub = (evtParamLen >= 1) ? params[0] : 0xFF;
                if (sub == 0x02) {
                    g_intelBootup = true;
                } else if (sub == 0x06 && evtParamLen >= 5) {
                    g_secureResult = params[1];
                    g_secureStatus = params[4];
                    g_secureResultValid = true;
                    // A healthy bootloader stays silent until the final
                    // fragment, so the byte/frag position here pins exactly
                    // where it reacted -- and a non-zero result/status mid
                    // upload is the signature of an active rejection.
                    bool err = (params[1] != 0) || (params[4] != 0);
                    KernelLogStream(err ? ERROR : INFO, "BT-HCI") << "secure-send result="
                        << base::hex << (uint64_t)params[1] << " status="
                        << (uint64_t)params[4] << base::dec
                        << " @ byte " << g_ssBytesSent
                        << " frag " << (uint64_t)g_ssFragsSent;
                }
                break;
            }

            default:
                break;
        }
    }

    // =========================================================================
    // ProcessAcl — handle incoming ACL data
    // =========================================================================

    void ProcessAcl(const uint8_t* data, uint32_t len) {
        if (len < sizeof(AclHeader)) return;

        auto* hdr = (const AclHeader*)data;
        uint16_t handle = hdr->HandleFlags & 0x0FFF;
        uint16_t pbFlag = hdr->HandleFlags & 0x3000;
        uint16_t dataLen = hdr->DataLength;

        if (dataLen + sizeof(AclHeader) > len) return;

        // Dispatch to L2CAP
        L2cap::ProcessPacket(handle, data + sizeof(AclHeader), dataLen);
    }

    // =========================================================================
    // Connection management
    // =========================================================================

    ConnectionInfo* GetConnection(uint16_t handle) {
        for (int i = 0; i < MAX_CONNECTIONS; i++) {
            if (g_connections[i].Active && g_connections[i].Handle == handle) {
                return &g_connections[i];
            }
        }
        return nullptr;
    }

    ConnectionInfo* GetActiveConnection() {
        for (int i = 0; i < MAX_CONNECTIONS; i++) {
            if (g_connections[i].Active) {
                return &g_connections[i];
            }
        }
        return nullptr;
    }

    ConnectionInfo* GetConnectionByIndex(int index) {
        if (index < 0 || index >= MAX_CONNECTIONS) return nullptr;
        return &g_connections[index];
    }

    // =========================================================================
    // Convenience HCI commands
    // =========================================================================

    bool Reset() {
        if (!SendCommand(OP_RESET, nullptr, 0)) return false;
        BusyWaitMs(100);
        return WaitCommandComplete(OP_RESET, nullptr, 0, 5000);
    }

    bool ReadBdAddr(uint8_t* addr) {
        if (!SendCommand(OP_READ_BD_ADDR, nullptr, 0)) return false;
        uint8_t params[7] = {};
        if (!WaitCommandComplete(OP_READ_BD_ADDR, params, sizeof(params))) return false;
        // params[0] = status, params[1..6] = BD_ADDR
        if (params[0] != 0) return false;
        memcpy(addr, &params[1], 6);
        return true;
    }

    bool ReadLocalVersion(LocalVersion* ver) {
        if (!SendCommand(OP_READ_LOCAL_VERSION, nullptr, 0)) return false;
        uint8_t params[9] = {};
        if (!WaitCommandComplete(OP_READ_LOCAL_VERSION, params, sizeof(params))) return false;
        if (params[0] != 0) return false;
        if (ver) {
            ver->Status = params[0];
            ver->HciVersion = params[1];
            ver->HciRevision = (uint16_t)params[2] | ((uint16_t)params[3] << 8);
            ver->LmpVersion = params[4];
            ver->Manufacturer = (uint16_t)params[5] | ((uint16_t)params[6] << 8);
            ver->LmpSubversion = (uint16_t)params[7] | ((uint16_t)params[8] << 8);
        }
        return true;
    }

    bool ReadIntelVersion(IntelVersion* ver) {
        // Newer Intel BT controllers (AX200/AX201/AX211, THP+) require a
        // parameter byte of 0xFF for 0xFC05 to return the full version in
        // TLV format.  Try with the parameter first; fall back to the
        // legacy (no-param) format if the command fails.
        uint8_t param = 0xFF;
        bool sent = SendCommand(OP_INTEL_READ_VERSION, &param, 1);
        if (!sent) {
            // Fallback: legacy format (no parameter)
            sent = SendCommand(OP_INTEL_READ_VERSION, nullptr, 0);
        }
        if (!sent) return false;

        uint8_t params[32] = {};
        if (!WaitCommandComplete(OP_INTEL_READ_VERSION, params, sizeof(params))) return false;

        // Log raw response for diagnostics
        KernelLogStream(INFO, "BT-HCI") << "Intel version raw: "
            << base::hex
            << (uint64_t)params[0] << " " << (uint64_t)params[1] << " "
            << (uint64_t)params[2] << " " << (uint64_t)params[3] << " "
            << (uint64_t)params[4] << " " << (uint64_t)params[5] << " "
            << (uint64_t)params[6] << " " << (uint64_t)params[7] << " "
            << (uint64_t)params[8] << " " << (uint64_t)params[9]
            << base::dec;

        if (ver) memcpy(ver, params, sizeof(IntelVersion));
        return true;
    }

    // =========================================================================
    // Intel firmware download primitives
    // =========================================================================

    int ReadIntelVersionTlv(uint8_t* outBuf, int maxLen) {
        if (!outBuf || maxLen <= 0) return -1;

        uint8_t param = 0xFF;
        uint32_t completionsBefore = g_intInCompletions.load(std::memory_order_relaxed);

        // Up to 3 attempts, 150ms apart: a bootloader that dropped one FC05
        // (busy, or still flushing a previous multi-packet response) gets a
        // fresh chance instead of failing the whole bring-up.
      for (int attempt = 0; attempt < 3; attempt++) {
        if (attempt > 0) {
            KernelLogStream(WARNING, "BT-HCI") << "Retrying Intel TLV version read (attempt "
                << (uint64_t)(attempt + 1) << ")";
            PollWait(150);
        }
        if (!SendCommand(OP_INTEL_READ_VERSION, &param, 1)) return -1;

        // Wait for the matching Command Complete and copy the *actually
        // received* return parameters.  The TLV version response can exceed a
        // single interrupt packet; WaitCommandComplete trusts the event's
        // declared length, so we read g_eventBuf/g_eventLen directly to avoid
        // copying past what the controller delivered.
        uint64_t start = Timekeeping::GetMilliseconds();
        while (Timekeeping::GetMilliseconds() - start < 2000) {
            Xhci::PollEvents();

            if (g_eventReady) {
                g_eventReady = false;
                if (g_eventLen >= 5 && g_eventBuf[0] == EVT_COMMAND_COMPLETE) {
                    uint16_t op = (uint16_t)g_eventBuf[3] | ((uint16_t)g_eventBuf[4] << 8);
                    if (op == OP_INTEL_READ_VERSION) {
                        // Return params begin at byte 5 (status, then TLVs).
                        int avail = (int)g_eventLen - 5;
                        if (avail < 0) avail = 0;
                        int n = (avail < maxLen) ? avail : maxLen;
                        memcpy(outBuf, &g_eventBuf[5], n);
                        return n;
                    }
                }
            }

            for (int j = 0; j < 100; j++) asm volatile("" ::: "memory");
        }

        // Distinguish "pipe dead" (0 completions) from "responses arriving but
        // never matching" (mailbox scrambling) -- the two need different fixes.
        KernelLogStream(WARNING, "BT-HCI") << "ReadIntelVersionTlv timeout ("
            << (uint64_t)(g_intInCompletions.load(std::memory_order_relaxed) - completionsBefore)
            << " int-in completions during wait)";
        DumpFwTrace();
      }
        return -1;
    }

    void ClearSecureSendResult() {
        g_secureResultValid = false;
        g_ssBytesSent = 0;
        g_ssFragsSent = 0;
    }

    bool PeekSecureSendResult(uint8_t* outResult, uint8_t* outStatus) {
        if (!g_secureResultValid) return false;
        if (outResult) *outResult = g_secureResult;
        if (outStatus) *outStatus = g_secureStatus;
        return true;
    }

    void ProcessPendingCommands() {
        // Top-level only: drains queued pairing replies with real, confirmed
        // control transfers.  Must NOT be called from inside PollEvents.
        if (Xhci::InPollContext()) return;
        // One consumer at a time: this runs from the Connect wait loop AND the
        // idle-core service pump, potentially on different cores; two consumers
        // on the lockless queue would race the tail and interleave commands.
        static std::atomic<bool> s_active{false};
        bool expected = false;
        if (!s_active.compare_exchange_strong(expected, true,
                                              std::memory_order_acquire)) return;
        while (g_pendingTail != g_pendingHead) {
            PendingHciCmd c = g_pending[g_pendingTail];
            g_pendingTail = (uint8_t)((g_pendingTail + 1) & 15);
            SendCommand(c.opcode, c.params, c.len);
            // Set Connection Encryption returns Command Status (not Complete),
            // so wait on that instead of burning a 1s timeout that delays A2DP.
            if (c.opcode == OP_SET_CONN_ENCRYPT) {
                WaitCommandStatus(c.opcode, 1000);
            } else {
                WaitCommandComplete(c.opcode, nullptr, 0, 1000);
            }
        }
        s_active.store(false, std::memory_order_release);
    }

    bool WaitSecureSendResult(uint32_t timeoutMs, uint8_t* outResult, uint8_t* outStatus) {
        uint64_t start = Timekeeping::GetMilliseconds();
        while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
            Xhci::PollEvents();
            if (g_secureResultValid) {
                if (outResult) *outResult = g_secureResult;
                if (outStatus) *outStatus = g_secureStatus;
                return true;
            }
            for (int j = 0; j < 100; j++) asm volatile("" ::: "memory");
        }
        return false;
    }

    bool IntelSecureSend(uint8_t fragmentType, const uint8_t* data, uint32_t len) {
        uint32_t off = 0;
        while (len > 0) {
            uint8_t frag = (len > 252) ? 252 : (uint8_t)len;

            // Fragment: [type][up to 252 data bytes]
            uint8_t buf[253];
            buf[0] = fragmentType;
            memcpy(&buf[1], data + off, frag);

            // The Intel bootloader does NOT return a Command Complete for
            // 0xFC09 (Linux's btusb injects a fake one).  Pacing comes from the
            // synchronous USB transfer in SendCommand; the download outcome is
            // reported asynchronously via the 0xFF/0x06 secure-send result
            // event.  So: send, drain events, and move on -- do not block per
            // fragment waiting for a reply that never arrives.
            if (!SendCommand(OP_INTEL_SECURE_SEND, buf, (uint8_t)(frag + 1))) {
                KernelLogStream(ERROR, "BT-HCI") << "Secure send transport error: type="
                    << base::hex << (uint64_t)fragmentType << " cc=" << (uint64_t)g_lastControlCC
                    << base::dec << " at frag #" << (uint64_t)g_ssFragsSent
                    << " byte " << g_ssBytesSent;
                return false;
            }
            Xhci::PollEvents();

            g_ssBytesSent += frag;
            g_ssFragsSent++;

            len -= frag;
            off += frag;
        }
        return true;
    }

    // Bulk-OUT variant of IntelSecureSend, for the firmware PAYLOAD phase.
    // In bootloader mode the Intel controller accepts 0xFC09 command packets
    // on the bulk OUT endpoint (the btusb bootloader path); unlike EP0's
    // 3-stage synchronous transfers this lets several fragments be in flight
    // at once, which is where Linux's ~1.3s download speed comes from.
    // Reuses the ACL TX DMA ring (no ACL traffic can exist yet -- the ACL
    // header is deliberately absent, and the NOCP credit counter is NOT
    // touched: these are commands, not ACL data).
    bool IntelSecureSendBulk(uint8_t fragmentType, const uint8_t* data, uint32_t len) {
        if (!g_initialized || !g_aclTxRing[0]) return false;

        uint32_t off = 0;
        while (len > 0) {
            uint8_t frag = (len > 252) ? 252 : (uint8_t)len;

            // Throttle on TX ring slots (leave one of headroom), reaping
            // completions while full.  A wall-clock bound keeps a wedged
            // controller from freezing the boot path.
            uint64_t start = Timekeeping::GetMilliseconds();
            while (AclTxInFlight() >= (uint32_t)(ACL_TX_SLOTS - 1)) {
                Xhci::PollEvents();
                if (Timekeeping::GetMilliseconds() - start > 2000) {
                    KernelLogStream(ERROR, "BT-HCI")
                        << "Bulk secure-send stalled at frag #" << (uint64_t)g_ssFragsSent
                        << " byte " << g_ssBytesSent;
                    return false;
                }
                asm volatile("pause" ::: "memory");
            }

            g_aclTxLock.Acquire();
            uint8_t* txBuf = g_aclTxRing[g_aclTxSlot];
            uint64_t txPhys = g_aclTxRingPhys[g_aclTxSlot];
            g_aclTxSlot = (uint8_t)((g_aclTxSlot + 1) % ACL_TX_SLOTS);

            // Raw HCI command packet, bulk-framed: opcode + plen + [type + data].
            txBuf[0] = (uint8_t)(OP_INTEL_SECURE_SEND & 0xFF);
            txBuf[1] = (uint8_t)(OP_INTEL_SECURE_SEND >> 8);
            txBuf[2] = (uint8_t)(frag + 1);
            txBuf[3] = fragmentType;
            memcpy(&txBuf[4], data + off, frag);

            g_aclTxCount.fetch_add(1, std::memory_order_relaxed);
            Xhci::QueueBulkOutTransfer(g_slotId, txBuf, txPhys, (uint32_t)frag + 4);
            g_aclTxLock.Release();

            g_ssBytesSent += frag;
            g_ssFragsSent++;
            len -= frag;
            off += frag;
        }
        return true;
    }

    // Wait until every queued bulk OUT transfer has completed (all payload
    // bytes handed to the controller).  Call before waiting for the final
    // secure-send result.
    bool DrainBulkTx(uint32_t timeoutMs) {
        uint64_t start = Timekeeping::GetMilliseconds();
        while (AclTxInFlight() > 0) {
            Xhci::PollEvents();
            if (Timekeeping::GetMilliseconds() - start > timeoutMs) return false;
            asm volatile("pause" ::: "memory");
        }
        return true;
    }

    bool IntelBootFirmware(uint32_t bootAddr, uint32_t timeoutMs) {
        g_intelBootup = false;

        // intel_reset: reset_type=0x00, patch_enable=0x01, ddc_reload=0x00,
        // boot_option=0x01 (boot at specified address), boot_param (LE32).
        uint8_t params[8] = {
            0x00, 0x01, 0x00, 0x01,
            (uint8_t)(bootAddr & 0xFF),
            (uint8_t)((bootAddr >> 8) & 0xFF),
            (uint8_t)((bootAddr >> 16) & 0xFF),
            (uint8_t)((bootAddr >> 24) & 0xFF),
        };

        // Fire and forget: the controller reboots into operational firmware
        // and signals readiness via the Intel bootup vendor event rather than
        // a Command Complete for 0xFC01.
        if (!SendCommand(OP_INTEL_RESET, params, sizeof(params))) return false;

        uint64_t start = Timekeeping::GetMilliseconds();
        while (Timekeeping::GetMilliseconds() - start < timeoutMs) {
            Xhci::PollEvents();
            if (g_intelBootup) return true;
            for (int j = 0; j < 100; j++) asm volatile("" ::: "memory");
        }

        KernelLogStream(ERROR, "BT-HCI") << "Timed out waiting for Intel bootup event";
        return false;
    }

    bool IntelWriteDdcRecord(const uint8_t* record, uint8_t recordLen) {
        if (!record || recordLen == 0) return false;
        if (!SendCommand(OP_INTEL_DDC_CONFIG_WRITE, record, recordLen)) return false;
        uint8_t st[4] = {};
        if (!WaitCommandComplete(OP_INTEL_DDC_CONFIG_WRITE, st, sizeof(st), 2000)) return false;
        return st[0] == 0;
    }

    bool IntelSetEventMask() {
        // Enables the Intel vendor events used during/after firmware load.
        uint8_t mask[8] = { 0x87, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
        if (!SendCommand(OP_INTEL_SET_EVENT_MASK, mask, sizeof(mask))) return false;
        return WaitCommandComplete(OP_INTEL_SET_EVENT_MASK, nullptr, 0, 2000);
    }

    bool WriteLocalName(const char* name) {
        uint8_t params[248] = {};
        int i = 0;
        for (; i < 247 && name[i]; i++) params[i] = name[i];
        params[i] = '\0';
        if (!SendCommand(OP_WRITE_LOCAL_NAME, params, 248)) return false;
        return WaitCommandComplete(OP_WRITE_LOCAL_NAME);
    }

    bool WriteClassOfDevice(uint32_t cod) {
        uint8_t params[3] = {
            (uint8_t)(cod & 0xFF),
            (uint8_t)((cod >> 8) & 0xFF),
            (uint8_t)((cod >> 16) & 0xFF)
        };
        if (!SendCommand(OP_WRITE_CLASS_OF_DEVICE, params, 3)) return false;
        return WaitCommandComplete(OP_WRITE_CLASS_OF_DEVICE);
    }

    bool WriteScanEnable(uint8_t mode) {
        if (!SendCommand(OP_WRITE_SCAN_ENABLE, &mode, 1)) return false;
        return WaitCommandComplete(OP_WRITE_SCAN_ENABLE);
    }

    bool WriteSSPMode(uint8_t mode) {
        if (!SendCommand(OP_WRITE_SSP_MODE, &mode, 1)) return false;
        return WaitCommandComplete(OP_WRITE_SSP_MODE);
    }

    bool AcceptConnection(const uint8_t* bdAddr, uint8_t role) {
        uint8_t params[7];
        memcpy(params, bdAddr, 6);
        params[6] = role;
        if (!SendCommand(OP_ACCEPT_CONN_REQ, params, 7)) return false;
        return WaitCommandStatus(OP_ACCEPT_CONN_REQ);
    }

    bool AuthenticateLink(uint16_t handle) {
        // Request authentication on an established ACL link.  As the connection
        // initiator we must drive this: a bonded headset will not start pairing
        // on its own for a device it believes it already knows, so the link just
        // sits unauthenticated until it drops (reason 0x05).  This kicks off the
        // flow -- the controller raises a Link Key Request (we answer with a
        // stored key, or negatively to force fresh Secure Simple Pairing).
        uint8_t params[2] = { (uint8_t)(handle & 0xFF), (uint8_t)(handle >> 8) };
        if (!SendCommand(OP_AUTH_REQUESTED, params, 2)) return false;
        return WaitCommandStatus(OP_AUTH_REQUESTED, 2000);
    }

    bool SetBdAddr(const uint8_t* addr) {
        // Intel 0xFC31: 6-byte BD_ADDR, little-endian (same order as ReadBdAddr).
        if (!SendCommand(OP_INTEL_WRITE_BD_ADDR, addr, 6)) return false;
        return WaitCommandComplete(OP_INTEL_WRITE_BD_ADDR, nullptr, 0, 2000);
    }

    bool Disconnect(uint16_t handle, uint8_t reason) {
        uint8_t params[3] = {
            (uint8_t)(handle & 0xFF),
            (uint8_t)((handle >> 8) & 0xFF),
            reason
        };
        if (!SendCommand(OP_DISCONNECT, params, 3)) return false;
        return WaitCommandStatus(OP_DISCONNECT);
    }

    bool ReadBufferSize(uint16_t* aclLen, uint8_t* scoLen,
                        uint16_t* aclNum, uint16_t* scoNum) {
        if (!SendCommand(OP_READ_BUFFER_SIZE, nullptr, 0)) return false;
        uint8_t params[8] = {};
        if (!WaitCommandComplete(OP_READ_BUFFER_SIZE, params, sizeof(params))) return false;
        if (params[0] != 0) return false;
        if (aclLen) *aclLen = (uint16_t)params[1] | ((uint16_t)params[2] << 8);
        if (scoLen) *scoLen = params[3];
        if (aclNum) *aclNum = (uint16_t)params[4] | ((uint16_t)params[5] << 8);
        if (scoNum) *scoNum = (uint16_t)params[6] | ((uint16_t)params[7] << 8);
        g_aclMaxLen = (uint16_t)params[1] | ((uint16_t)params[2] << 8);
        g_aclMaxNum = (uint16_t)params[4] | ((uint16_t)params[5] << 8);
        return true;
    }

    // =========================================================================
    // Inquiry (device discovery)
    // =========================================================================

    bool StartInquiry(uint8_t durationUnits) {
        g_inquiryResultCount = 0;
        g_inquiryActive = true;

        // HCI Inquiry: LAP(3) + InquiryLength(1) + NumResponses(1)
        // GIAC LAP = 0x9E8B33
        uint8_t params[5] = {
            0x33, 0x8B, 0x9E,   // LAP (General Inquiry Access Code)
            durationUnits,       // Duration in 1.28s units
            0x00                 // Unlimited responses
        };

        if (!SendCommand(OP_INQUIRY, params, 5)) {
            g_inquiryActive = false;
            return false;
        }

        // Inquiry uses Command Status (not Command Complete)
        if (!WaitCommandStatus(OP_INQUIRY)) {
            g_inquiryActive = false;
            return false;
        }

        return true;
    }

    bool CancelInquiry() {
        if (!g_inquiryActive) return true;
        if (!SendCommand(OP_INQUIRY_CANCEL, nullptr, 0)) return false;
        WaitCommandComplete(OP_INQUIRY_CANCEL, nullptr, 0, 2000);
        g_inquiryActive = false;
        return true;
    }

    int GetInquiryResults(InquiryDevice* buf, int maxCount) {
        int count = g_inquiryResultCount;
        if (count > maxCount) count = maxCount;
        if (buf && count > 0) {
            memcpy(buf, g_inquiryResults, count * sizeof(InquiryDevice));
        }
        return count;
    }

    void ClearInquiryResults() {
        g_inquiryResultCount = 0;
    }

    bool IsInquiryActive() {
        return g_inquiryActive;
    }

    // =========================================================================
    // Create ACL connection
    // =========================================================================

    void DrainEvents() {
        // One drainer at a time.  This is called from every BT wait loop, the
        // audio-write pump (syscall context, any core) and the idle-core
        // service pump; the RX ring is single-consumer.  Two cores inside the
        // loop below would process the same slot twice (the duplicate
        // Connection Request seen on HW) and advance the tail past unprocessed
        // slots (the lost AVDTP Start response).  Skipping is correct: the
        // active drainer processes everything queued.
        static std::atomic<bool> s_draining{false};
        bool expected = false;
        if (!s_draining.compare_exchange_strong(expected, true,
                                                std::memory_order_acquire)) return;

        // Discard any unconsumed Command Complete/Status events that weren't
        // picked up by WaitCommandComplete/WaitCommandStatus.
        if (g_eventReady) {
            g_eventReady = false;
        }

        // Drain all queued ACL packets (process the whole ring, not just one,
        // so a burst is never left unhandled).
        while (g_aclRxTail != g_aclRxHead) {
            uint8_t slot = g_aclRxTail;
            uint16_t pl = g_aclRxLens[slot];

            // Diagnostic (top-level, safe to log -- not the IRQ path): trace the
            // first few ACL packets so the rx flood / missing Config Response is
            // identifiable in ONE boot.  Layout: ACL header(4) + L2CAP header(4),
            // so the L2CAP CID is at bytes 6-7; on the signaling channel (CID 1)
            // byte 8 is the command code (0x03 CONN_RSP, 0x04 CONFIG_REQ,
            // 0x05 CONFIG_RSP).  A flood of len==maxpacket junk CIDs vs real
            // cid=0001 code=05 packets tells the two failure modes apart.
            static uint32_t s_rxTraced = 0;
            if (s_rxTraced < 24 && pl >= 8) {
                const uint8_t* d = g_aclRxRing[slot];
                uint16_t cid = (uint16_t)d[6] | ((uint16_t)d[7] << 8);
                uint8_t code = (pl >= 9) ? d[8] : 0;
                KernelLogStream(INFO, "BT-HCI") << "rx[" << (uint64_t)s_rxTraced
                    << "] len=" << (uint64_t)pl << " cid=" << base::hex << (uint64_t)cid
                    << " code=" << (uint64_t)code << base::dec;
                s_rxTraced++;
            }

            ProcessAcl(g_aclRxRing[slot], pl);
            g_aclRxTail = (uint8_t)((g_aclRxTail + 1) % ACL_RX_SLOTS);
        }

        s_draining.store(false, std::memory_order_release);
    }

    bool CreateConnection(const uint8_t* bdAddr) {
        // HCI Create Connection:
        // BD_ADDR(6) + PacketType(2) + PSRM(1) + reserved(1) + ClockOffset(2) + AllowRoleSwitch(1)
        uint8_t params[13] = {};
        memcpy(params, bdAddr, 6);
        // Packet types: DM1, DH1, DM3, DH3, DM5, DH5
        params[6] = 0x18;  // CC18 = allow DM1, DH1, DM3, DH3, DM5, DH5
        params[7] = 0xCC;
        params[8] = 0x02;  // Page Scan Repetition Mode R2
        params[9] = 0x00;  // Reserved
        params[10] = 0x00; // Clock offset
        params[11] = 0x00;
        params[12] = 0x01; // Allow role switch

        if (!SendCommand(OP_CREATE_CONNECTION, params, 13)) return false;

        // Create Connection uses Command Status, then Connection Complete event
        return WaitCommandStatus(OP_CREATE_CONNECTION, 5000);
    }

}
