feat: aarch64 port - Raspberry Pi 4/400 SD boot (SDHCI, ext2 root + first-boot grow), USB hubs, PCI INTx routing, SMP, DMA fences

kernel/src/Arch/aarch64/Cpu.hpp

@@ -26,6 +26,15 @@ namespace Arch {
         asm volatile("dsb sy" ::: "memory");
     }
 
+    // Devices are outer-shareable observers.
+    inline void DmaWriteFence() {
+        asm volatile("dmb oshst" ::: "memory");
+    }
+
+    inline void DmaReadFence() {
+        asm volatile("dmb oshld" ::: "memory");
+    }
+
     inline void EnableInterrupts() {
         asm volatile("msr daifclr, #2" ::: "memory");
     }
@@ -93,13 +102,38 @@ namespace Arch {
     }
 
     // The TLBI operand is VA[55:12] in bits [43:0]; the bits above are RES0.
+    inline uint64_t DataCacheLineSize() {
+        uint64_t ctr;
+        asm volatile("mrs %0, ctr_el0" : "=r"(ctr));
+        return 4ULL << ((ctr >> 16) & 0xF);  // DminLine
+    }
+
+    // To the point of coherency: where non-snooping device DMA looks.
+    inline void CleanDataCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = (uint64_t)address & ~(line - 1); p < end; p += line) {
+            asm volatile("dc cvac, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb sy" ::: "memory");
+    }
+
+    // Clean too: a partial line at either end may hold CPU data next to the
+    // buffer, which a plain invalidate would throw away.
+    inline void InvalidateDataCache(const void* address, uint64_t size) {
+        uint64_t line = DataCacheLineSize();
+        uint64_t end = (uint64_t)address + size;
+        for (uint64_t p = (uint64_t)address & ~(line - 1); p < end; p += line) {
+            asm volatile("dc civac, %0" :: "r"(p) : "memory");
+        }
+        asm volatile("dsb sy" ::: "memory");
+    }
+
     // Clean the data cache to the point of unification, then invalidate
     // the whole instruction cache: invalidating by address would only
     // reach the alias it names on an aliasing (VIPT) instruction cache.
     inline void SyncInstructionCache(const void* address, uint64_t size) {
-        uint64_t ctr;
-        asm volatile("mrs %0, ctr_el0" : "=r"(ctr));
-        uint64_t line = 4ULL << ((ctr >> 16) & 0xF);  // DminLine
+        uint64_t line = DataCacheLineSize();
         uint64_t start = (uint64_t)address & ~(line - 1);
         uint64_t end = (uint64_t)address + size;
         for (uint64_t p = start; p < end; p += line) {