Kernel: Add Processor::flush_data_cache()

This function flushes (cleans + invalidates) the data cache(s) for the
given address range.

On x86, this function only works if CLFLUSH is supported. If we need it,
we could add a WBINVD fallback in the future.
However, CLFLUSH should be widely supported since it was introduced
with SSE2 (but not all processors with SSE2 are guaranteed to have it).

On RISC-V, this function requires Zicbom. Hardware that was released
before that extension was ratified will need hardware-specific
implementations, if we want to support them.

Kernel/Arch/Processor.h

@@ -63,6 +63,8 @@ public:
     static void flush_tlb_local(VirtualAddress vaddr, size_t page_count);
     static void flush_tlb(Memory::PageDirectory const*, VirtualAddress, size_t);
 
+    // FIXME: Add wrappers for data cache clean or invalidate operations when necessary.
+    static void flush_data_cache(VirtualAddress vaddr, size_t byte_count);
     static void flush_instruction_cache(VirtualAddress vaddr, size_t byte_count);
 
     void early_initialize(u32 cpu);

Kernel/Arch/aarch64/Processor.cpp

@@ -9,6 +9,7 @@
 #include <AK/Vector.h>
 
 #include <Kernel/Arch/Interrupts.h>
+#include <Kernel/Arch/MemoryFences.h>
 #include <Kernel/Arch/Processor.h>
 #include <Kernel/Arch/TrapFrame.h>
 #include <Kernel/Arch/aarch64/ASM_wrapper.h>
@@ -100,6 +101,18 @@ void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress vaddr
     flush_tlb_local(vaddr, page_count);
 }
 
+void ProcessorBase::flush_data_cache(VirtualAddress vaddr, size_t byte_count)
+{
+    // NOTE: This assumes that all processors have the same cache line size,
+    //       as a context switch can happen during execution of this function.
+
+    auto const cache_line_size = Aarch64::Asm::get_cache_line_size();
+    for (FlatPtr addr = align_down_to(vaddr.get(), cache_line_size); addr < vaddr.get() + byte_count; addr += cache_line_size)
+        asm volatile("dc civac, %0" ::"r"(addr) : "memory");
+
+    full_memory_fence();
+}
+
 void ProcessorBase::flush_instruction_cache(VirtualAddress vaddr, size_t byte_count)
 {
     __builtin___clear_cache(reinterpret_cast<char*>(vaddr.as_ptr()), reinterpret_cast<char*>(vaddr.offset(byte_count).as_ptr()));

Kernel/Arch/riscv64/Processor.cpp

@@ -243,6 +243,28 @@ void ProcessorBase::flush_tlb(Memory::PageDirectory const*, VirtualAddress vaddr
     flush_tlb_local(vaddr, page_count);
 }
 
+void ProcessorBase::flush_data_cache(VirtualAddress vaddr, size_t byte_count)
+{
+    if (!Processor::current().has_feature(CPUFeature::Zicbom))
+        return;
+
+    // Flushing userspace memory requires sstatus.SUM=1.
+    SmapDisabler disabler;
+
+    // NOTE: This assumes that all processors have Zicbom and the same m_zicbom_block_size,
+    //       as a context switch can happen during execution of this function.
+
+    auto const block_size = current().m_zicbom_block_size;
+    for (FlatPtr addr = align_down_to(vaddr.get(), block_size); addr < vaddr.get() + byte_count; addr += block_size) {
+        asm volatile(R"(
+            .option push
+            .option arch, +zicbom
+                cbo.flush (%0)
+            .option pop
+        )" ::"r"(addr) : "memory");
+    }
+}
+
 void ProcessorBase::flush_instruction_cache(VirtualAddress, size_t)
 {
     // FIXME: Use the SBI RFENCE extension to flush the instruction cache of other harts when we support SMP on riscv64.
@@ -690,6 +712,16 @@ void Processor::find_and_parse_devicetree_node()
 
         generate_userspace_extension_bitmask();
 
+        if (has_feature(CPUFeature::Zicbom)) {
+            auto zicbom_block_size = cpu.get_property("riscv,cbom-block-size"sv);
+            if (zicbom_block_size.has_value() && zicbom_block_size->size() == sizeof(u32))
+                m_zicbom_block_size = zicbom_block_size->as<u32>();
+            else
+                dmesgln("CPU[{}]: Supports Zicbom but its node ({}) doesn't have a valid \"riscv,cbom-block-size\" property", id(), cpu_name);
+        } else {
+            dmesgln("CPU[{}]: Doesn't support Zicbom, Processor::flush_data_cache() won't work", id());
+        }
+
         break;
     }
 }

Kernel/Arch/riscv64/Processor.h

@@ -74,6 +74,8 @@ private:
 
     Optional<ProcessorInfo> m_info;
     Array<unsigned long long, EXTENSION_BITMASK_GROUP_COUNT> m_userspace_extension_bitmask {};
+
+    u32 m_zicbom_block_size { 0 };
 };
 
 ALWAYS_INLINE bool ProcessorBase::is_initialized()

Kernel/Arch/x86_64/Processor.cpp

@@ -23,6 +23,7 @@
 #include <Kernel/Tasks/Thread.h>
 
 #include <Kernel/Arch/Interrupts.h>
+#include <Kernel/Arch/MemoryFences.h>
 #include <Kernel/Arch/Processor.h>
 #include <Kernel/Arch/SafeMem.h>
 #include <Kernel/Arch/TrapFrame.h>
@@ -229,6 +230,9 @@ UNMAP_AFTER_INIT void Processor::cpu_detect()
     if (processor_info.edx() & (1 << 31))
         m_features |= CPUFeature::PBE;
 
+    if (has_feature(CPUFeature::CLFLUSH))
+        m_clflush_cache_line_size = ((processor_info.ebx() >> 8) & 0xff) * 8;
+
     CPUID extended_features(0x7);
 
     if (extended_features.ebx() & (1 << 0))
@@ -667,6 +671,9 @@ UNMAP_AFTER_INIT void ProcessorBase::initialize(u32 cpu)
     if (self->m_has_qemu_hvf_quirk.was_set())
         dmesgln("CPU[{}]: Applied correction for QEMU Hypervisor.framework quirk", current_id());
 
+    if (!has_feature(CPUFeature::CLFLUSH))
+        dmesgln("CPU[{}]: Doesn't support CLFUSH, Processor::flush_data_cache() won't work", current_id());
+
     if (cpu == 0)
         initialize_interrupts();
     else
@@ -807,6 +814,26 @@ void ProcessorBase::flush_tlb(Memory::PageDirectory const* page_directory, Virtu
         flush_tlb_local(vaddr, page_count);
 }
 
+void ProcessorBase::flush_data_cache(VirtualAddress vaddr, size_t byte_count)
+{
+    // FIXME: Implement a WBINVD fallback. WBINVD only flushes the caches for the current processor,
+    //        so we need to broadcast an SMP messsage to do that.
+    // FIXME: Maybe use CLFLUSHOPT, if supported.
+    if (!Processor::current().has_feature(CPUFeature::CLFLUSH))
+        return;
+
+    // NOTE: This assumes that all processors have CLFLUSH and the same m_clflush_cache_line_size,
+    //       as a context switch can happen during execution of this function.
+
+    full_memory_fence();
+
+    auto const cache_line_size = current().m_clflush_cache_line_size;
+    for (FlatPtr addr = align_down_to(vaddr.get(), cache_line_size); addr < vaddr.get() + byte_count; addr += cache_line_size)
+        asm volatile("clflush %0" ::"m"(addr) : "memory");
+
+    full_memory_fence();
+}
+
 void ProcessorBase::flush_instruction_cache(VirtualAddress, size_t)
 {
     // The instruction and data cache are coherent on x86, so we don't need to do anything here.

Kernel/Arch/x86_64/Processor.h

@@ -67,6 +67,8 @@ private:
     alignas(Descriptor) Descriptor m_gdt[256];
     u32 m_gdt_length;
 
+    u16 m_clflush_cache_line_size { 0 };
+
     static Atomic<u32> s_idle_cpu_mask;
 
     TSS m_tss;