Reference counting + proper queues + SICI 5.11 support

.gitignore

@@ -32,3 +32,8 @@ modules.order
 
 # Ignore build directory
 build/*
+
+# Ignore cscope files
+cscope.out
+cscope.in.out
+cscope.po.out

CMakeLists.txt

@@ -33,7 +33,12 @@ foreach (arch ${nvidia_archs})
     string (CONCAT GPU_ARCHS ${GPU_ARCHS} " -gencode arch=compute_${arch},code=sm_${arch}")
 endforeach ()
 
-set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra" CACHE INTERNAL "" FORCE)
+set (convert_warnings implicit-function-declaration incompatible-pointer-types int-conversion return-type unused-function unused-variable)
+set (flags "")
+foreach (flag ${convert_warnings})
+    string (APPEND flags " -Werror=${flag}")
+endforeach()
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra ${flags}" CACHE INTERNAL "" FORCE)
 set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_DEBUG} -g -DDEBUG" CACHE INTERNAL "" FORCE)
 set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2" CACHE INTERNAL "" FORCE)
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra" CACHE INTERNAL "" FORCE)
@@ -75,13 +80,14 @@ endif ()
 find_path (sisci_include NAMES sisci_types.h sisci_api.h sisci_error.h PATHS "${DIS}/include" "${DIS}")
 find_library (sisci_lib NAMES sisci PATHS "${DIS}/lib64" "${DIS}/lib" NO_DEFAULT_PATH)
 if (sisci_lib)
-    message(STATUS "Using DIS library ${sisci_lib}")
+    message (STATUS "Using SISCI library ${sisci_lib}")
 endif ()
 
 # Library source files
 set (libnvm_root "${PROJECT_SOURCE_DIR}/src")
 file (GLOB libnvm_source "${libnvm_root}/*.c")
 file (GLOB libnvm_dis_source "${libnvm_root}/dis/*.c")
+file (GLOB libnvm_linux_source "${libnvm_root}/linux/*.c")
 file (GLOB libnvm_include "${PROJECT_BINARY_DIR}/include/*.h" "${PROJECT_SOURCE_DIR}/include/*.h")
 
 # Module source files
@@ -111,17 +117,19 @@ endif ()
 
 
 # Create NVME library target
-add_library (libnvm ${libnvm_source})
-target_include_directories (libnvm PRIVATE "${libnvm_root}" "${libnvm_root}/dis")
+add_library (libnvm ${libnvm_source} ${libnvm_linux_source}) # TODO: add check for OS
+target_include_directories (libnvm PRIVATE "${libnvm_root}")
 
 
 if (sisci_include AND sisci_lib AND NOT no_smartio)
     target_sources (libnvm PRIVATE ${libnvm_dis_source})
     target_compile_definitions (libnvm PRIVATE _REENTRANT _SISCI __DIS_CLUSTER__)
     target_link_libraries (libnvm ${sisci_lib} Threads::Threads)
+    message (STATUS "Configuring libnvm with SmartIO")
 
 else ()
     target_link_libraries (libnvm Threads::Threads)
+    message (STATUS "Configuring libnvm without SmartIO")
 
 endif ()
 
@@ -142,11 +150,11 @@ install (TARGETS libnvm DESTINATION "lib")
 # Create kernel module makefile
 if (NOT no_module)
     if (CUDA_FOUND AND NOT no_cuda AND EXISTS "${driver_dir}/Module.symvers")
-        set (module_ccflags "-D_CUDA -I${driver_include} -I${libnvm_root}")
+        set (module_ccflags "-D_CUDA -I${driver_include} -I${libnvm_root} -I${libnvm_root}/linux")
         set (module_symbols "${driver_dir}/Module.symvers")
         message(STATUS "Configuring kernel module with CUDA")
     else ()
-        set (module_ccflags "-I${libnvm_root}")
+        set (module_ccflags "-I${libnvm_root} -I${libnvm_root}/linux")
         message(STATUS "Configuring kernel module without CUDA")
     endif ()
 

examples/identify/common.c

@@ -71,7 +71,7 @@ nvm_aq_ref reset_ctrl(const nvm_ctrl_t* ctrl, const nvm_dma_t* dma_window)
     {
         return NULL;
     }
-    memset(dma_window->vaddr, 0, dma_window->page_size * 2);
+    memset((void*) dma_window->vaddr, 0, dma_window->page_size * 2);
 
     fprintf(stderr, "Resetting controller and setting up admin queues...\n");
     status = nvm_aq_create(&admin, ctrl, dma_window);

examples/identify/userspace.c

@@ -114,7 +114,7 @@ static int identify(const nvm_ctrl_t* ctrl, uint32_t nvm_ns_id)
         goto out;
     }
 
-    status = nvm_dma_map(&window, ctrl, memory, page_size, 3, ioaddrs);
+    status = nvm_raw_dma_map(&window, ctrl, memory, page_size, 3, ioaddrs);
     if (status != 0)
     {
         fprintf(stderr, "Failed to create DMA window: %s\n", strerror(status));

include/nvm_admin.h

@@ -53,13 +53,14 @@ int nvm_admin_request_num_queues(nvm_aq_ref ref, uint16_t* n_cqs, uint16_t* n_sq
 /*
  * Create IO completion queue (CQ)
  * Caller must set queue memory to zero manually.
+ * If n_pages > 1, DMA memory must be contiguous.
  */
-int nvm_admin_cq_create(nvm_aq_ref ref,               // AQ pair reference
-                        nvm_queue_t* cq,              // CQ descriptor
-                        uint16_t id,                  // Queue identifier
-                        void* qmem,                   // Queue memory (virtual memory)
-                        uint64_t ioaddr);             // Bus address to queue memory as seen by controller
-
+int nvm_admin_cq_create(nvm_aq_ref ref,                 // AQ pair reference
+                        nvm_queue_t* cq,                // CQ descriptor
+                        uint16_t id,                    // Queue identifier
+                        const nvm_dma_t* dma,           // Queue memory handle
+                        size_t page_offset,             // Number of pages to offset into the handle
+                        size_t n_pages);                // Number of pages to use 
 
 /*
  * Delete IO completion queue (CQ)
@@ -73,13 +74,15 @@ int nvm_admin_cq_delete(nvm_aq_ref ref, nvm_queue_t* cq);
 /*
  * Create IO submission queue (SQ)
  * Caller must set queue memory to zero manually.
+ * If n_pages > 1, DMA memory must be contiguous.
  */
-int nvm_admin_sq_create(nvm_aq_ref ref,               // AQ pair reference
-                        nvm_queue_t* sq,              // SQ descriptor
-                        const nvm_queue_t* cq,        // Descriptor to paired CQ
-                        uint16_t id,                  // Queue identifier
-                        void* qmem,                   // Queue memory (virtual)
-                        uint64_t ioaddr);             // Bus address to queue as seen by controller
+int nvm_admin_sq_create(nvm_aq_ref ref,                 // AQ pair reference
+                        nvm_queue_t* sq,                // SQ descriptor
+                        const nvm_queue_t* cq,          // Descriptor to paired CQ
+                        uint16_t id,                    // Queue identifier
+                        const nvm_dma_t* dma,           // Queue memory handle
+                        size_t page_offset,             // Number of pages to offset into the handle
+                        size_t n_pages);                // Number of pages to use
 
 
 
@@ -92,6 +95,16 @@ int nvm_admin_sq_delete(nvm_aq_ref ref,
                         const nvm_queue_t* cq);
 
 
+/*
+ * Get log page.
+ */
+int nvm_admin_get_log_page(nvm_aq_ref ref, 
+                           uint32_t ns_id, 
+                           void* ptr, 
+                           uint64_t ioaddr, 
+                           uint8_t log_id, 
+                           uint64_t log_offset);
+
 #ifdef __cplusplus
 }
 #endif

include/nvm_aq.h

@@ -19,8 +19,9 @@ extern "C" {
  *
  * Returns a reference handle that can be used for admin RPC calls.
  */
-int nvm_aq_create(nvm_aq_ref* ref, const nvm_ctrl_t* ctrl, const nvm_dma_t* aq_mem);
-
+int nvm_aq_create(nvm_aq_ref* ref, 
+                  const nvm_ctrl_t* ctrl, 
+                  const nvm_dma_t* dma_window);
 
 
 /*
@@ -44,6 +45,7 @@ void nvm_aq_destroy(nvm_aq_ref ref);
 
 #ifdef __DIS_CLUSTER__
 
+
 /*
  * Maximum number of adapters.
  */

include/nvm_cmd.h

@@ -10,42 +10,42 @@
 #include <nvm_types.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
 
 
 
 /* All namespaces identifier */
 #define NVM_CMD_NS_ALL                  0xffffffff
+#define NVM_CMD_OPCODE(gen, fun, data)  (_WB((gen), 7, 7) | _WB((fun), 6, 2) | _WB((data), 1, 0))
 
 
 /* List of NVM IO command opcodes */
 enum nvm_io_command_set
 {
-    NVM_IO_FLUSH                    = (0x00 << 7) | (0x00 << 2) | 0x00, // 00h
-    NVM_IO_WRITE                    = (0x00 << 7) | (0x00 << 2) | 0x01, // 01h
-    NVM_IO_READ                     = (0x00 << 7) | (0x00 << 2) | 0x02, // 02h
-    NVM_IO_WRITE_ZEROES             = (0x00 << 7) | (0x02 << 2) | 0x00  // 08h
+    NVM_IO_FLUSH            = NVM_CMD_OPCODE(0, 0, 0),  // 00h
+    NVM_IO_WRITE            = NVM_CMD_OPCODE(0, 0, 1),  // 01h
+    NVM_IO_READ             = NVM_CMD_OPCODE(0, 0, 2),  // 02h
+    NVM_IO_WRITE_ZEROES     = NVM_CMD_OPCODE(0, 2, 0)   // 08h
 };
 
 
 
 /* List of NVM admin command opcodes */
 enum nvm_admin_command_set
 {
-    NVM_ADMIN_DELETE_SUBMISSION_QUEUE   = (0x00 << 7) | (0x00 << 2) | 0x00,
-    NVM_ADMIN_CREATE_SUBMISSION_QUEUE   = (0x00 << 7) | (0x00 << 2) | 0x01,
-    NVM_ADMIN_DELETE_COMPLETION_QUEUE   = (0x00 << 7) | (0x01 << 2) | 0x00,
-    NVM_ADMIN_CREATE_COMPLETION_QUEUE   = (0x00 << 7) | (0x01 << 2) | 0x01,
-    NVM_ADMIN_IDENTIFY                  = (0x00 << 7) | (0x01 << 2) | 0x02,
-    NVM_ADMIN_GET_LOG_PAGE              = (0x00 << 7) | (0x00 << 2) | 0x02,
-    NVM_ADMIN_ABORT                     = (0x00 << 7) | (0x02 << 2) | 0x00,
-    NVM_ADMIN_SET_FEATURES              = (0x00 << 7) | (0x02 << 2) | 0x01,
-    NVM_ADMIN_GET_FEATURES              = (0x00 << 7) | (0x02 << 2) | 0x02
+    NVM_ADMIN_DELETE_SQ     = NVM_CMD_OPCODE(0, 0, 0),  // 00h
+    NVM_ADMIN_CREATE_SQ     = NVM_CMD_OPCODE(0, 0, 1),  // 01h
+    NVM_ADMIN_GET_LOG_PAGE  = NVM_CMD_OPCODE(0, 0, 2),  // 02h
+    NVM_ADMIN_DELETE_CQ     = NVM_CMD_OPCODE(0, 1, 0),  // 04h
+    NVM_ADMIN_CREATE_CQ     = NVM_CMD_OPCODE(0, 1, 1),  // 05h
+    NVM_ADMIN_IDENTIFY      = NVM_CMD_OPCODE(0, 1, 2),  // 06h
+    NVM_ADMIN_ABORT         = NVM_CMD_OPCODE(0, 2, 0),  // 08h
+    NVM_ADMIN_SET_FEATURES  = NVM_CMD_OPCODE(0, 2, 1),  // 09h
+    NVM_ADMIN_GET_FEATURES  = NVM_CMD_OPCODE(0, 2, 2)   // 0Ah
 };
 
 
 
-
-
 /*
  * Set command's DWORD0 and DWORD1
  */
@@ -100,50 +100,90 @@ void nvm_cmd_rw_blks(nvm_cmd_t* cmd, uint64_t start_lba, uint16_t n_blks)
 
 
 /*
- * Build a PRP list consisting of PRP entries.
+ * Build PRP list consisting of PRP entries.
  *
  * Populate a memory page with PRP entries required for a transfer.
- * Returns the number of PRP entries used. Number of pages should 
- * always be max_data_size (MDTS) for IO commands.
+ * If the number of pages exceed the available number of entries, the last
+ * entry will not be used (so that it can be used to point to the next list).
+ * (See Chapter 4.4, Figure 14 in the NVMe specification).
  *
- * Note: currently, PRP lists can only be a single page
+ * Returns the number of PRP entries used. 
  */
 __host__ __device__ static inline
-size_t nvm_prp_list(size_t page_size, size_t n_pages, void* list_ptr, const uint64_t* data_ioaddrs)
+size_t nvm_prp_list(const nvm_prp_list_t* list, size_t n_pages, const uint64_t* ioaddrs)
 {
-    // TODO #ifdef __NO_COHERENCE__, make a nvm_prp_list_far variant that does not call nvm_cache_flush()
-    size_t prps_per_page = page_size / sizeof(uint64_t);
+    size_t n_prps = list->page_size / sizeof(uint64_t);
     size_t i_prp;
-    uint64_t* list;
-
-    if (prps_per_page < n_pages)
+    volatile uint64_t* entries = (volatile uint64_t*) list->vaddr;
+
+    // Do we need to reserve the last entry for the next list?
+    if (n_pages > n_prps)
     {
-        n_pages = prps_per_page;
+        --n_prps;
     }
-
-    list = (uint64_t*) list_ptr;
-    for (i_prp = 0; i_prp < n_pages; ++i_prp)
+    else
+    {
+        n_prps = n_pages;
+    }
+
+    // Populate list
+    for (i_prp = 0; i_prp < n_prps; ++i_prp)
     {
-        list[i_prp] = data_ioaddrs[i_prp];
+        entries[i_prp] = ioaddrs[i_prp];
     }
 
-    nvm_cache_flush(list_ptr, sizeof(uint64_t) * i_prp);
+    // Flush list cache
+    if (list->local)
+    {
+        nvm_cache_flush((void*) list->vaddr, sizeof(uint64_t) * i_prps);
+    }
+    else
+    {
+        nvm_wcb_flush();
+    }
 
     return i_prp;
 }
 
 
 
+/*
+ * Build chain of PRP lists.
+ * Returns the total number of PRP entries.
+ */
+__host__ __device__ static inline
+size_t nvm_prp_list_chain(size_t n_lists, const nvm_prp_list_t* lists, size_t n_pages, const uint64_t* ioaddrs)
+{
+    size_t i_list;
+    size_t list_prps;
+    size_t n_prps;
+
+    if (n_lists == 0 || lists == NULL)
+    {
+        return 0;
+    }
+
+    list_prps = nvm_prp_list(&lists[0], n_pages, ioaddrs);
+
+    for (i_list = 1, n_prps = list_prps; i_list < n_lists && n_prps < n_pages; ++i_list, n_prps += list_prps)
+    {
+        volatile uint64_t* next_list_ptr = ((volatile uint64_t*) lists[i_list - 1].vaddr) + list_prps;
+        *next_list_ptr = lists[i_list].ioaddr;
+
+        list_prps = nvm_prp_list(&lists[i_list], n_pages - n_prps, &ioaddrs[n_prps]);
+    }
+
+    return n_prps;
+}
+
+
+
 /*
  * Helper function to build a PRP list and set a command's data pointer fields.
+ * Number of pages should always be max_data_size (MDTS) for IO commands.
  */
 __host__ __device__ static inline
-size_t nvm_cmd_data(nvm_cmd_t* cmd, 
-                    size_t page_size, 
-                    size_t n_pages, 
-                    void* list_ptr, 
-                    uint64_t list_ioaddr, 
-                    const uint64_t* data_ioaddrs)
+size_t nvm_cmd_data(nvm_cmd_t* cmd, size_t n_lists, const nvm_prp_list_t* lists, size_t n_pages, const uint64_t* ioaddrs)
 {
     size_t prp = 0;
     uint64_t dptr0 = 0;
@@ -155,17 +195,21 @@ size_t nvm_cmd_data(nvm_cmd_t* cmd,
         return 0;
     }
 #endif
+    if (lists == NULL)
+    {
+        n_lists = 0;
+    }
 
-    dptr0 = data_ioaddrs[prp++];
+    dptr0 = ioaddrs[prp++];
 
-    if (n_pages > 2 && list_ptr != NULL)
+    if (n_pages > 2 && n_lists != 0)
     {
-        dptr1 = list_ioaddr;
-        prp += nvm_prp_list(page_size, n_pages - 1, list_ptr, &data_ioaddrs[prp]);
+        prp += nvm_prp_list_chain(n_lists, lists, n_pages - 1, &ioaddrs[prp]);
+        dptr1 = lists[0].ioaddr;
     }
     else if (n_pages >= 2)
     {
-        dptr1 = data_ioaddrs[prp++];
+        dptr1 = ioaddrs[prp++];
     }
 
     nvm_cmd_data_ptr(cmd, dptr0, dptr1);