Kernel: Use MemoryType::IO for DMA regions everywhere

Drivers using DMA buffers are currently broken on bare metal since
d3a0ae5 and b3bae90 made DMA buffers use the NonCacheable memory
type.

We should investigate each of these drivers and and add proper fences
where needed.

The only place where MemoryType::IO definitely isn't needed is the xHCI
scratchpad regions, as they are only accessed by the device.

Kernel/Bus/USB/UHCI/UHCIController.cpp

@@ -109,7 +109,8 @@ ErrorOr<void> UHCIController::reset()
     }
 
     // Let's allocate the physical page for the Frame List (which is 4KiB aligned)
-    m_framelist = TRY(MM.allocate_dma_buffer_page("UHCI Framelist"sv, Memory::Region::Access::Write));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    m_framelist = TRY(MM.allocate_dma_buffer_page("UHCI Framelist"sv, Memory::Region::Access::Write, Memory::MemoryType::IO));
     dbgln("UHCI: Allocated framelist at physical address {}", m_framelist->physical_page(0)->paddr());
     dbgln("UHCI: Framelist is at virtual address {}", m_framelist->vaddr());
     write_sofmod(64); // 1mS frame time
@@ -145,7 +146,8 @@ UNMAP_AFTER_INIT ErrorOr<void> UHCIController::create_structures()
     // Now the Transfer Descriptor pool
     m_transfer_descriptor_pool = TRY(UHCIDescriptorPool<TransferDescriptor>::try_create("Transfer Descriptor Pool"sv));
 
-    m_isochronous_transfer_pool = TRY(MM.allocate_dma_buffer_page("UHCI Isochronous Descriptor Pool"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    m_isochronous_transfer_pool = TRY(MM.allocate_dma_buffer_page("UHCI Isochronous Descriptor Pool"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
 
     // Set up the Isochronous Transfer Descriptor list
     m_iso_td_list.resize(UHCI_NUMBER_OF_ISOCHRONOUS_TDS);

Kernel/Bus/USB/USBPipe.cpp

@@ -34,7 +34,8 @@ ErrorOr<void> Pipe::clear_halt()
 
 ErrorOr<NonnullOwnPtr<ControlPipe>> ControlPipe::create(USBController const& controller, Device& device, u8 endpoint_number, u16 max_packet_size, size_t buffer_size)
 {
-    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB device DMA buffer"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB device DMA buffer"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     return adopt_nonnull_own_or_enomem(new (nothrow) ControlPipe(controller, device, endpoint_number, max_packet_size, move(dma_buffer)));
 }
 
@@ -74,7 +75,8 @@ ErrorOr<size_t> ControlPipe::submit_control_transfer(u8 request_type, u8 request
 ErrorOr<NonnullOwnPtr<BulkInPipe>> BulkInPipe::create(USBController const& controller, Device& device, u8 endpoint_number, u16 max_packet_size, size_t buffer_size)
 {
     VERIFY(buffer_size >= max_packet_size);
-    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     return adopt_nonnull_own_or_enomem(new (nothrow) BulkInPipe(controller, device, endpoint_number, max_packet_size, move(dma_buffer)));
 }
 
@@ -120,7 +122,8 @@ ErrorOr<size_t> BulkInPipe::submit_bulk_in_transfer(size_t length, UserOrKernelB
 ErrorOr<NonnullOwnPtr<BulkOutPipe>> BulkOutPipe::create(USBController const& controller, Device& device, u8 endpoint_number, u16 max_packet_size, size_t buffer_size)
 {
     VERIFY(buffer_size >= max_packet_size);
-    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     return adopt_nonnull_own_or_enomem(new (nothrow) BulkOutPipe(controller, device, endpoint_number, max_packet_size, move(dma_buffer)));
 }
 
@@ -166,7 +169,8 @@ ErrorOr<size_t> BulkOutPipe::submit_bulk_out_transfer(size_t length, UserOrKerne
 ErrorOr<NonnullOwnPtr<InterruptInPipe>> InterruptInPipe::create(USBController const& controller, Device& device, u8 endpoint_number, u16 max_packet_size, u16 poll_interval, size_t buffer_size)
 {
     VERIFY(buffer_size >= max_packet_size);
-    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     return adopt_nonnull_own_or_enomem(new (nothrow) InterruptInPipe(controller, device, endpoint_number, max_packet_size, poll_interval, move(dma_buffer)));
 }
 
@@ -189,7 +193,8 @@ ErrorOr<NonnullLockRefPtr<Transfer>> InterruptInPipe::submit_interrupt_in_transf
 ErrorOr<NonnullOwnPtr<InterruptOutPipe>> InterruptOutPipe::create(USBController const& controller, Device& device, u8 endpoint_number, u16 max_packet_size, u16 poll_interval, size_t buffer_size)
 {
     VERIFY(buffer_size >= max_packet_size);
-    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    auto dma_buffer = TRY(MM.allocate_dma_buffer_pages(TRY(Memory::page_round_up(buffer_size)), "USB pipe DMA buffer"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     return adopt_nonnull_own_or_enomem(new (nothrow) InterruptOutPipe(controller, device, endpoint_number, max_packet_size, poll_interval, move(dma_buffer)));
 }
 

Kernel/Bus/USB/xHCI/xHCIController.cpp

@@ -216,7 +216,8 @@ ErrorOr<void> xHCIController::initialize()
     m_operational_registers.configure.max_device_slots_enabled = m_device_slots;
 
     // 2. Program the Device Context Base Address Array Pointer (DCBAAP) register (5.4.6) with a 64-bit address pointing to where the Device Context Base Address Array is located.
-    m_device_context_base_address_array_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up((m_device_slots + 1) * sizeof(u64))), "xHCI Device Context Base Address Array"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    m_device_context_base_address_array_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up((m_device_slots + 1) * sizeof(u64))), "xHCI Device Context Base Address Array"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     dbgln_if(XHCI_DEBUG, "xHCI: Device Context Base Address Array - {} / {}", m_device_context_base_address_array_region->vaddr(), m_device_context_base_address_array_region->physical_page(0)->paddr());
     m_device_context_base_address_array = reinterpret_cast<u64*>(m_device_context_base_address_array_region->vaddr().as_ptr());
     auto requested_scratchpad_buffers = (m_capability_registers.structural_parameters.max_scratchpad_buffers_high << 5) | m_capability_registers.structural_parameters.max_scratchpad_buffers_low;
@@ -238,7 +239,8 @@ ErrorOr<void> xHCIController::initialize()
     m_operational_registers.device_context_base_address_array_pointer.high = device_context_base_address_array_pointer >> 32;
 
     // 3. Define the Command Ring Dequeue Pointer by programming the Command Ring Control Register (5.4.5) with a 64-bit address pointing to the starting address of the first TRB of the Command Ring.
-    m_command_and_event_rings_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(sizeof(CommandAndEventRings))), "xHCI Command and Event Rings"sv, Memory::Region::Access::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    m_command_and_event_rings_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(sizeof(CommandAndEventRings))), "xHCI Command and Event Rings"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
     dbgln_if(XHCI_DEBUG, "xHCI: Command and Event Rings - {} / {}", m_command_and_event_rings_region->vaddr(), m_command_and_event_rings_region->physical_page(0)->paddr());
     auto command_and_event_rings_region_virtual_address = m_command_and_event_rings_region->vaddr().get();
     m_command_ring = reinterpret_cast<TransferRequestBlock*>(command_and_event_rings_region_virtual_address + __builtin_offsetof(CommandAndEventRings, command_ring));
@@ -545,7 +547,8 @@ ErrorOr<void> xHCIController::initialize_device(USB::Device& device)
 
     // 5. After successfully obtaining a Device Slot, system software shall initialize the data structures associated with the slot as described in section 4.3.3.
     //   1. Allocate an Input Context data structure (6.2.5) and initialize all fields to ‘0’.
-    slot_state.input_context_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(input_context_size())), "xHCI Input Context"sv, Memory::Region::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    slot_state.input_context_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(input_context_size())), "xHCI Input Context"sv, Memory::Region::ReadWrite, Memory::MemoryType::IO));
 
     //   2. Initialize the Input Control Context (6.2.5.1) of the Input Context by setting the A0 and A1 flags to ‘1’.
     // These flags indicate that the Slot Context and the Endpoint 0 Context of the Input Context are affected by the command.
@@ -623,7 +626,8 @@ ErrorOr<void> xHCIController::initialize_device(USB::Device& device)
 
     //   4.  Allocate and initialize the Transfer Ring for the Default Control Endpoint.
     //   Refer to section 4.9 for TRB Ring initialization requirements and to section 6.4 for the formats of TRBs
-    slot_state.endpoint_rings[0].region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(endpoint_ring_size * sizeof(TransferRequestBlock))), "xHCI Endpoint Rings"sv, Memory::Region::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    slot_state.endpoint_rings[0].region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(endpoint_ring_size * sizeof(TransferRequestBlock))), "xHCI Endpoint Rings"sv, Memory::Region::ReadWrite, Memory::MemoryType::IO));
     auto* endpoint_ring_memory = slot_state.endpoint_rings[0].ring_vaddr();
     endpoint_ring_memory[endpoint_ring_size - 1].generic.transfer_request_block_type = TransferRequestBlock::TRBType::Link;
     auto endpoint_ring_address = slot_state.endpoint_rings[0].ring_paddr();
@@ -671,7 +675,8 @@ ErrorOr<void> xHCIController::initialize_device(USB::Device& device)
     endpoint_context->average_transfer_request_block = 8;
 
     //   6. Allocate the Output Device Context data structure (6.2.1) and initialize it to ‘0’.
-    slot_state.device_context_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(device_context_size())), "xHCI Device Context"sv, Memory::Region::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    slot_state.device_context_region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(device_context_size())), "xHCI Device Context"sv, Memory::Region::ReadWrite, Memory::MemoryType::IO));
 
     //   7. Load the appropriate (Device Slot ID) entry in the Device Context Base Address Array (5.4.6) with a pointer to the Output Device Context data structure (6.2.1).
     m_device_context_base_address_array[slot] = slot_state.device_context_region->physical_page(0)->paddr().get();
@@ -1000,7 +1005,8 @@ ErrorOr<void> xHCIController::initialize_endpoint_if_needed(Pipe const& pipe)
     if (endpoint_ring.region)
         return {}; // Already initialized
 
-    endpoint_ring.region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(endpoint_ring_size * sizeof(TransferRequestBlock))), "xHCI Endpoint Rings"sv, Memory::Region::ReadWrite));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    endpoint_ring.region = TRY(MM.allocate_dma_buffer_pages(MUST(Memory::page_round_up(endpoint_ring_size * sizeof(TransferRequestBlock))), "xHCI Endpoint Rings"sv, Memory::Region::ReadWrite, Memory::MemoryType::IO));
     auto* endpoint_ring_memory = endpoint_ring.ring_vaddr();
     endpoint_ring_memory[endpoint_ring_size - 1].generic.transfer_request_block_type = TransferRequestBlock::TRBType::Link;
     auto endpoint_ring_address = endpoint_ring.ring_paddr();

Kernel/Devices/Audio/AC97/AC97.cpp

@@ -224,13 +224,16 @@ ErrorOr<size_t> AC97::write(size_t channel_index, UserOrKernelBuffer const& data
     if (channel_index != 0)
         return Error::from_errno(ENODEV);
 
-    if (!m_output_buffer)
-        m_output_buffer = TRY(MM.allocate_dma_buffer_pages(m_output_buffer_page_count * PAGE_SIZE, "AC97 Output buffer"sv, Memory::Region::Access::Write));
+    if (!m_output_buffer) {
+        // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+        m_output_buffer = TRY(MM.allocate_dma_buffer_pages(m_output_buffer_page_count * PAGE_SIZE, "AC97 Output buffer"sv, Memory::Region::Access::Write, Memory::MemoryType::IO));
+    }
 
     if (!m_buffer_descriptor_list) {
         size_t buffer_descriptor_list_size = buffer_descriptor_list_max_entries * sizeof(BufferDescriptorListEntry);
         buffer_descriptor_list_size = TRY(Memory::page_round_up(buffer_descriptor_list_size));
-        m_buffer_descriptor_list = TRY(MM.allocate_dma_buffer_pages(buffer_descriptor_list_size, "AC97 Buffer Descriptor List"sv, Memory::Region::Access::Write));
+        // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+        m_buffer_descriptor_list = TRY(MM.allocate_dma_buffer_pages(buffer_descriptor_list_size, "AC97 Buffer Descriptor List"sv, Memory::Region::Access::Write, Memory::MemoryType::IO));
     }
 
     Checked<size_t> remaining = length;

Kernel/Devices/Audio/IntelHDA/RingBuffer.h

@@ -45,7 +45,8 @@ class ControllerRingBuffer {
 
         // Create a DMA buffer page to holds the ring buffer
         VERIFY(PAGE_SIZE >= capacity * sizeof(T));
-        auto buffer_region = TRY(MM.allocate_dma_buffer_page(name, U == RingBufferType::Input ? Memory::Region::Access::Read : Memory::Region::Access::Write));
+        // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+        auto buffer_region = TRY(MM.allocate_dma_buffer_page(name, U == RingBufferType::Input ? Memory::Region::Access::Read : Memory::Region::Access::Write, Memory::MemoryType::IO));
 
         // 4.4.1.1, 4.4.2: The CORB buffer in memory must be allocated to start on a 128-byte boundary
         // and in memory configured to match the access type being used.

Kernel/Devices/Audio/IntelHDA/Stream.cpp

@@ -67,7 +67,8 @@ ErrorOr<void> Stream::initialize_buffer()
     size_t cyclic_buffer_size_in_bytes = number_of_buffers_required_for_cyclic_buffer_size * PAGE_SIZE;
 
     TRY(m_buffers.with([&](auto& buffers) -> ErrorOr<void> {
-        buffers = TRY(MM.allocate_dma_buffer_pages(cyclic_buffer_size_in_bytes, "IntelHDA Stream Buffers"sv, Memory::Region::Access::ReadWrite));
+        // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+        buffers = TRY(MM.allocate_dma_buffer_pages(cyclic_buffer_size_in_bytes, "IntelHDA Stream Buffers"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
 
         // 3.3.38 Input/Output/Bidirectional Stream Descriptor Cyclic Buffer Length
         m_stream_io_window->write32(StreamRegisterOffset::CyclicBufferLength, buffers->size());
@@ -76,7 +77,8 @@ ErrorOr<void> Stream::initialize_buffer()
         m_stream_io_window->write16(StreamRegisterOffset::LastValidIndex, number_of_buffers_required_for_cyclic_buffer_size - 1);
 
         // 3.6.2: Buffer Descriptor List
-        m_buffer_descriptor_list = TRY(MM.allocate_dma_buffer_page("IntelHDA Stream BDL"sv, Memory::Region::Access::ReadWrite));
+        // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+        m_buffer_descriptor_list = TRY(MM.allocate_dma_buffer_page("IntelHDA Stream BDL"sv, Memory::Region::Access::ReadWrite, Memory::MemoryType::IO));
         auto bdl_physical_address = m_buffer_descriptor_list->physical_page(0)->paddr().get();
         m_stream_io_window->write32(StreamRegisterOffset::BDLLowerBaseAddress, bdl_physical_address & 0xffffffffu);
         m_stream_io_window->write32(StreamRegisterOffset::BDLUpperBaseAddress, bdl_physical_address >> 32);

Kernel/Devices/Storage/AHCI/Port.cpp

@@ -46,7 +46,8 @@ ErrorOr<void> AHCIPort::allocate_resources_and_initialize_ports()
         m_command_table_pages.append(move(command_table_page));
     }
 
-    m_command_list_region = TRY(MM.allocate_dma_buffer_page("AHCI Port Command List"sv, Memory::Region::Access::ReadWrite, m_command_list_page));
+    // FIXME: Synchronize DMA buffer accesses correctly and set the MemoryType to NonCacheable.
+    m_command_list_region = TRY(MM.allocate_dma_buffer_page("AHCI Port Command List"sv, Memory::Region::Access::ReadWrite, m_command_list_page, Memory::MemoryType::IO));
 
     dbgln_if(AHCI_DEBUG, "AHCI Port {}: Command list page at {}", representative_port_index(), m_command_list_page->paddr());
     dbgln_if(AHCI_DEBUG, "AHCI Port {}: FIS receive page at {}", representative_port_index(), m_fis_receive_page->paddr());