Set config and enable dsa accel module offload. This feature is consi…

…dered as experimental. Signed-off-by: Alexander Indenbaum <[email protected]>
ceph · Dec 17, 2024 · 87b9650 · 87b9650
1 parent bf83ae5
commit 87b9650
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 47 deletions.
diff --git a/.env b/.env
@@ -57,7 +57,7 @@ SPDK_URL="https://spdk.io"
 
 SPDK_PKGDEP_ARGS="--rbd"
 # check spdk/configure --help
-SPDK_CONFIGURE_ARGS="--with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug"
+SPDK_CONFIGURE_ARGS="--with-idxd --with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug"
 SPDK_TARGET_ARCH="x86-64-v2"
 SPDK_MAKEFLAGS=
 SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/"

diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -86,47 +86,47 @@ jobs:
           path: |
             ceph.tar
 
-  build-arm64:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Build container images - spdk
-        run: make build SVC=spdk TARGET_ARCH=arm64
-
-      - name: Build container images - bdevperf
-        run: make build SVC=bdevperf TARGET_ARCH=arm64
-
-      - name: Build container images - nvmeof
-        run: make build SVC=nvmeof TARGET_ARCH=arm64
-
-      - name: Build container images - nvmeof-cli
-        run: make build SVC=nvmeof-cli TARGET_ARCH=arm64
-
-      - name: Build container images - ceph
-        run: make build SVC=ceph TARGET_ARCH=arm64
-
-      - name: Save container images
-        run: |
-          . .env
-          docker tag $QUAY_NVMEOF:$NVMEOF_VERSION $QUAY_NVMEOF:$NVMEOF_VERSION-arm64
-          docker tag $QUAY_NVMEOFCLI:$NVMEOF_VERSION $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64
-          docker save $QUAY_NVMEOF:$NVMEOF_VERSION-arm64 > nvmeof-arm64.tar
-          docker save $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64 > nvmeof-cli-arm64.tar
-
-      - name: Upload nvmeof-arm64 container images
-        uses: actions/upload-artifact@v4
-        with:
-          name: container_images_nvmeof_arm64
-          path: |
-            nvmeof-arm64.tar
-            nvmeof-cli-arm64.tar
+# build-arm64:
+#   runs-on: ubuntu-latest
+#   steps:
+#     - name: Checkout code
+#       uses: actions/checkout@v4
+#       with:
+#         submodules: recursive
+
+#     - name: Set up QEMU
+#       uses: docker/setup-qemu-action@v3
+
+#     - name: Build container images - spdk
+#       run: make build SVC=spdk TARGET_ARCH=arm64
+
+#     - name: Build container images - bdevperf
+#       run: make build SVC=bdevperf TARGET_ARCH=arm64
+
+#     - name: Build container images - nvmeof
+#       run: make build SVC=nvmeof TARGET_ARCH=arm64
+
+#     - name: Build container images - nvmeof-cli
+#       run: make build SVC=nvmeof-cli TARGET_ARCH=arm64
+
+#     - name: Build container images - ceph
+#       run: make build SVC=ceph TARGET_ARCH=arm64
+
+#     - name: Save container images
+#       run: |
+#         . .env
+#         docker tag $QUAY_NVMEOF:$NVMEOF_VERSION $QUAY_NVMEOF:$NVMEOF_VERSION-arm64
+#         docker tag $QUAY_NVMEOFCLI:$NVMEOF_VERSION $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64
+#         docker save $QUAY_NVMEOF:$NVMEOF_VERSION-arm64 > nvmeof-arm64.tar
+#         docker save $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64 > nvmeof-cli-arm64.tar
+
+#     - name: Upload nvmeof-arm64 container images
+#       uses: actions/upload-artifact@v4
+#       with:
+#         name: container_images_nvmeof_arm64
+#         path: |
+#           nvmeof-arm64.tar
+#           nvmeof-cli-arm64.tar
 
   pytest:
     needs: [build, build-ceph]
@@ -684,7 +684,7 @@ jobs:
 
   push-images-to-ceph-registry:
     if: github.event_name == 'release'
-    needs: [pytest, demo, discovery, ha, atom, build-arm64]
+    needs: [pytest, demo, discovery, ha, atom]
     runs-on: ubuntu-latest
 
     steps:
@@ -701,8 +701,8 @@ jobs:
         run: |
           docker load < nvmeof.tar
           docker load < nvmeof-cli.tar
-          docker load < nvmeof-arm64.tar
-          docker load < nvmeof-cli-arm64.tar
+          #docker load < nvmeof-arm64.tar
+          #docker load < nvmeof-cli-arm64.tar
 
       - name: Login to quay.io
         uses: docker/login-action@v2
@@ -714,7 +714,7 @@ jobs:
       - name: Publish nvmeof containers when release/tag is created
         run: |
           make push
-          make push TAG_SUFFIX="-arm64"
+          #make push TAG_SUFFIX="-arm64"
           make push-manifest-list
 
   push-devel-image-to-ceph-registry:

diff --git a/control/server.py b/control/server.py
@@ -23,6 +23,7 @@
 import spdk.rpc
 import spdk.rpc.client as rpc_client
 import spdk.rpc.nvmf as rpc_nvmf
+import spdk.rpc.dsa as rpc_dsa
 
 from .proto import gateway_pb2 as pb2
 from .proto import gateway_pb2_grpc as pb2_grpc
@@ -475,7 +476,7 @@ def _start_spdk(self, omap_state):
         self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}")
         spdk_tgt_cmd_extra_args = self.config.get_with_default(
             "spdk", "tgt_cmd_extra_args", "")
-        cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path]
+        cmd = [spdk_tgt_path, "--wait-for-rpc", "-u", "-r", self.spdk_rpc_socket_path]
 
         # Add extra args from the conf file
         if spdk_tgt_cmd_extra_args:
@@ -558,6 +559,9 @@ def _start_spdk(self, omap_state):
                 log_level=protocol_log_level,
                 conn_retries=conn_retries,
             )
+            # Set config and enable dsa accel module offload.
+            self._probe_dsa()
+
         except Exception:
             self.logger.exception(f"Unable to initialize SPDK")
             raise
@@ -653,6 +657,16 @@ def _stop_discovery(self):
 
         self.discovery_pid = None
 
+    def _probe_dsa(self):
+        """Initializes dsa accel module offload."""
+        try:
+            res = rpc_dsa.dsa_scan_accel_module(self.spdk_rpc_client)
+            spdk.rpc.framework_start_init(self.spdk_rpc_client)
+        except Exception:
+            self.logger.exception(f"Failed to probe dsa accel module offload")
+            raise
+        self.logger.debug(f"dsa_scan_accel_module: {res=}")
+
     def _create_transport(self, trtype):
         """Initializes a transport type."""
         args = {'trtype': trtype}