From 1ec57bf4af3931e9ba5dfc967de59939f6cc1dcb Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Tue, 28 May 2024 11:20:31 +0800 Subject: [PATCH 1/2] control/server.py: Enable DSA to offload and accelerate CRC calculation Signed-off-by: Yin Congmin --- .env | 2 +- README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++ ceph-nvmeof.conf | 3 +++ control/server.py | 9 +++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/.env b/.env index ad57be31..ca82e0d5 100644 --- a/.env +++ b/.env @@ -57,7 +57,7 @@ SPDK_URL="https://spdk.io" SPDK_PKGDEP_ARGS="--rbd" # check spdk/configure --help -SPDK_CONFIGURE_ARGS="--with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug" +SPDK_CONFIGURE_ARGS="--with-rbd --with-idxd --disable-tests --disable-unit-tests --disable-examples --enable-debug" SPDK_TARGET_ARCH="x86-64-v2" SPDK_MAKEFLAGS= SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/" diff --git a/README.md b/README.md index d1d419fa..ff54d850 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,72 @@ sh -c 'echo 4096 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages' This is automatically done in the `make setup` step. The amount of hugepages can be configured with `make setup HUGEPAGES=512`. +### Enable DSA to offload and accelerate CRC calculation + +Intel® Data Streaming Accelerator (Intel® DSA) can generate and test CRC checksum or Data Integrity Field (DIF) on the memory region to support usages typical with storage and networking applications. This feature has already been implemented in SPDK. Enabling this feature allows for offloading and accelerating CRC calculations in NVMe-oF. + +#### DSA Configuration + +```bash +[spdk] +enable_dsa = True +``` + +#### Enable DSA in containers using vfio driver + +Refer: [SPDK System Configuration User Guide: Device access](https://spdk.io/doc/system_configuration.html#system_configuration_nonroot_device_access). + +1. Load `vfio` and `vfio-pci` drivers. Enable `IOMMU`. + +2. Use DPDK to bind DSA as `vfio-pci`. In `ceph-nvmeof` directory, execute: + +```bash +./spdk/dpdk/usertools/dpdk-devbind.py -s +``` + +The output may include: + +```bash +DMA devices using DPDK-compatible driver +======================================== +0000:6a:01.0 'Device 0b25' drv=vfio-pci unused=idxd +0000:e7:01.0 'Device 0b25' drv=vfio-pci unused=idxd +``` + +If the driver is not `vfio-pci`, bind it to this. Execute: + +```bash +./spdk/dpdk/usertools/dpdk-devbind.py -u 0000:6a:01.0 0000:e7:01.0 +./spdk/dpdk/usertools/dpdk-devbind.py -b vfio-pci 0000:6a:01.0 0000:e7:01.0 +``` + +3. Check the IOMMU group of DSA devices: + +```bash +readlink "/sys/bus/pci/devices/0000:6a:01.0/iommu_group" +``` + +The output should be e.g. `../../../kernel/iommu_groups/49` + +4. update docker-compose.yaml: + +```bash + nvmeof-base: + devices: + - /dev/vfio/vfio:/dev/vfio/vfio + - /dev/vfio/49:/dev/vfio/49 + - /dev/vfio/250:/dev/vfio/250 + volumes: + cap_add: + - IPC_LOCK # DMA pinning +``` + +Then run `make up` to start container. If DSA is successfully enabled, the following logs can be seen: + +```bash +accel_dsa_rpc.c: 50:rpc_dsa_scan_accel_module: *NOTICE*: Enabled DSA user-mode +``` + ## Development ### Set-up diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf index 079bd7df..53101a62 100644 --- a/ceph-nvmeof.conf +++ b/ceph-nvmeof.conf @@ -66,6 +66,9 @@ tgt_path = /usr/local/bin/nvmf_tgt timeout = 60.0 #log_level = WARNING +# True / False to enable dsa +# enable_dsa = False + # Example value: -m 0x3 -L all # tgt_cmd_extra_args = diff --git a/control/server.py b/control/server.py index a038bf45..a35920eb 100644 --- a/control/server.py +++ b/control/server.py @@ -360,6 +360,10 @@ def _start_spdk(self, omap_state): spdk_tgt_cmd_extra_args = self.config.get_with_default( "spdk", "tgt_cmd_extra_args", "") cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path] + spdk_tgt_dsa = self.config.getboolean_with_default("spdk", "enable_dsa", False) + if spdk_tgt_dsa: + self.logger.info(f"Start SPDK, but wait for DSA detection before initialization") + cmd = [spdk_tgt_path, "-r", self.spdk_rpc_socket_path, "--wait-for-rpc"] if spdk_tgt_cmd_extra_args: cmd += shlex.split(spdk_tgt_cmd_extra_args) self.logger.info(f"Starting {' '.join(cmd)}") @@ -407,6 +411,11 @@ def _start_spdk(self, omap_state): log_level=protocol_log_level, conn_retries=conn_retries, ) + if spdk_tgt_dsa: + # init dsa + spdk.rpc.dsa.dsa_scan_accel_module(self.spdk_rpc_client) + spdk.rpc.framework_start_init(self.spdk_rpc_client) + self.logger.info(f"SPDK with DSA started") except Exception: self.logger.exception(f"Unable to initialize SPDK") raise From 610e74438fc67a068af0e9ae924f5eb6af680558 Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Mon, 2 Sep 2024 16:53:00 +0800 Subject: [PATCH 2/2] Update spdk submodule to gerrit-24172 Signed-off-by: Yin Congmin --- spdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spdk b/spdk index cd16195a..e54830fe 160000 --- a/spdk +++ b/spdk @@ -1 +1 @@ -Subproject commit cd16195a3dc914ea445e90b9cc8ba3211e17b267 +Subproject commit e54830fe9f803747d1e62b1ca5cdb0c872368327