Add support for TLS.

Fixes #646

Signed-off-by: Gil Bregman <[email protected]>

.github/workflows/build-container.yml

@@ -85,7 +85,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap", "log_files", "nsid"]
+        test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap", "log_files", "nsid", "psk"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 512  # for multi gateway test, approx 256 per gateway instance
@@ -344,6 +344,165 @@ jobs:
           make down
           make clean
 
+  demo-secure:
+    needs: [build, build-ceph]
+    runs-on: ubuntu-latest
+    env:
+      HUGEPAGES: 512
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup huge-pages
+        run: make setup HUGEPAGES=$HUGEPAGES
+
+      - name: Download container images
+        uses: actions/download-artifact@v4
+        with:
+          pattern: container_images*
+          merge-multiple: true
+
+      - name: Load container images
+        run: |
+          docker load < nvmeof.tar
+          docker load < nvmeof-cli.tar
+          docker load < ceph.tar
+          docker load < bdevperf.tar
+
+      - name: Start containers
+        timeout-minutes: 3
+        run: |
+          if ! docker-compose --version 2>&1 > /dev/null ; then
+            sudo apt update
+            sudo apt install -y docker-compose
+          fi
+          docker-compose --version
+          make up
+
+      - name: Wait for the Gateway to be listening
+        timeout-minutes: 3
+        run: |
+          . .env
+
+          echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT
+          until nc -z $NVMEOF_IP_ADDRESS $NVMEOF_GW_PORT; do
+            echo -n .
+            sleep ${{ env.WAIT_INTERVAL_SECS }}
+          done
+
+      - name: List containers
+        if: success() || failure()
+        run: make ps
+
+      - name: List processes
+        if: success() || failure()
+        run: make top
+
+      - name: Test
+        run: |
+          . .env
+          make demosecure OPTS=-T NVMEOF_CONTAINER_NAME="ceph-nvmeof_nvmeof_1" HOSTNQN="${NQN}host"
+          if [ ! -d "/var/lib/ceph/" ]; then
+              echo "/var/lib/ceph doesn't exit"
+              exit 1
+          fi
+
+      - name: List resources
+        run: |
+          # https://github.com/actions/toolkit/issues/766
+          shopt -s expand_aliases
+          eval $(make alias)
+          cephnvmf get_subsystems
+          cephnvmf subsystem list
+          subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//')
+          for sub in $subs
+          do
+              cephnvmf namespace list --subsystem $sub
+              cephnvmf listener list --subsystem $sub
+              cephnvmf host list --subsystem $sub
+          done
+
+      - name: Run bdevperf
+        run: |
+          # see https://spdk.io/doc/nvmf_multipath_howto.html
+          shopt -s expand_aliases
+          eval $(make alias)
+          . .env
+          set -x
+          echo -n "ℹ️  Starting bdevperf container"
+          docker-compose up -d bdevperf
+          sleep 10
+          echo "ℹ️  bdevperf start up logs"
+          make logs SVC=bdevperf
+          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )
+          echo "ℹ️  bdevperf find PSK key path"
+          fsid=$(make fsid)
+          psk_path="/var/lib/ceph/$fsid/psk/${NQN}host"
+          echo "ℹ️  bdevperf PSK key path is ${psk_path}"
+          if [ ! -f "$psk_path" ]; then
+              echo "PSK key $psk_path doesn't exit"
+              exit 1
+          fi
+          docker cp /var/lib/ceph ceph-nvmeof_bdevperf_1:/var/lib
+
+          rpc="/usr/libexec/spdk/scripts/rpc.py"
+          echo "ℹ️  bdevperf bdev_nvme_set_options"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
+          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10 --psk ${psk_path}"
+          echo "ℹ️  verify connection list"
+          conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN)
+          echo $conns | grep -q '"status": 0'
+          echo $conns | grep -q "\"nqn\": \"${NQN}host\""
+          echo $conns | grep -q "\"trsvcid\": ${NVMEOF_IO_PORT}"
+          echo $conns | grep -q "\"traddr\": \"${NVMEOF_IP_ADDRESS}\""
+          echo $conns | grep -q "\"adrfam\": \"ipv4\""
+          echo $conns | grep -q "\"trtype\": \"TCP\""
+          echo $conns | grep -q "\"qpairs_count\": 1"
+          echo $conns | grep -q "\"connected\": true"
+          echo $conns | grep -q "\"secure\": true"
+          echo $conns | grep -q "\"psk\": true"
+          con_cnt=$(echo $conns | xargs -n 1 | grep traddr | wc -l)
+          if [ $con_cnt -ne 1 ]; then
+              echo "Number of connections ${con_cnt}, expected 1. list: ${conns}"
+              exit 1
+          fi
+          echo "ℹ️  bdevperf perform_tests"
+          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
+          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
+          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
+          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"
+
+      - name: Check coredump existence
+        if: success() || failure()
+        id: check_coredumps
+        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b  # v2, pinned to SHA for security reasons
+        with:
+          files: "/tmp/coredump/core.*"
+
+      - name: Upload demo core dumps
+        if: steps.check_coredumps.outputs.files_exists == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: core_demo
+          path: /tmp/coredump/core.*
+
+      # For debugging purposes (provides an SSH connection to the runner)
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      #   with:
+      #     limit-access-to-actor: true
+
+      - name: Display logs
+        if: success() || failure()
+        run: make logs OPTS=''
+
+      - name: Tear down
+        if: success() || failure()
+        run: |
+          make down
+          make clean
+
   discovery:
     needs: [build, build-ceph]
     strategy:

Makefile

@@ -6,6 +6,7 @@ MAKEFLAGS += --no-builtin-rules --no-builtin-variables
 include .env
 include mk/containerized.mk
 include mk/demo.mk
+include mk/demosecure.mk
 include mk/misc.mk
 include mk/autohelp.mk
 
@@ -18,6 +19,9 @@ setup: ## Configure huge-pages (requires sudo/root password)
 	@echo Setup core dump pattern as /tmp/coredump/core.*
 	mkdir -p /tmp/coredump
 	sudo mkdir -p /var/log/ceph
+	sudo chmod 0777 /var/log/ceph
+	sudo mkdir -p /var/lib/ceph
+	sudo chmod 0777 /var/lib/ceph
 	sudo bash -c 'echo "|/usr/bin/env tee /tmp/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern'
 	sudo bash -c 'echo $(HUGEPAGES) > $(HUGEPAGES_DIR)'
 	@echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR))

ceph-nvmeof.conf

@@ -29,6 +29,7 @@ enable_spdk_discovery_controller = False
 #allowed_consecutive_spdk_ping_failures = 1
 #spdk_ping_interval_in_seconds = 2.0
 #ping_spdk_under_lock = False
+#psk_directory = /var/lib/ceph/security/psk
 
 [gateway-logs]
 log_level=debug

control/cli.py

@@ -855,6 +855,7 @@ def listener_add(self, args):
             adrfam=adrfam,
             traddr=traddr,
             trsvcid=args.trsvcid,
+            secure=args.secure,
         )
 
         try:
@@ -960,14 +961,15 @@ def listener_list(self, args):
                 for l in listeners_info.listeners:
                     adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, l.adrfam)
                     adrfam = self.format_adrfam(adrfam)
-                    listeners_list.append([l.host_name, l.trtype, adrfam, f"{l.traddr}:{l.trsvcid}"])
+                    secure = "Yes" if l.secure else "No"
+                    listeners_list.append([l.host_name, l.trtype, adrfam, f"{l.traddr}:{l.trsvcid}", secure])
                 if len(listeners_list) > 0:
                     if args.format == "text":
                         table_format = "fancy_grid"
                     else:
                         table_format = "plain"
                     listeners_out = tabulate(listeners_list,
-                                      headers = ["Host", "Transport", "Address Family", "Address"],
+                                      headers = ["Host", "Transport", "Address Family", "Address", "Secure"],
                                       tablefmt=table_format)
                     out_func(f"Listeners for {args.subsystem}:\n{listeners_out}")
                 else:
@@ -1000,6 +1002,7 @@ def listener_list(self, args):
         argument("--traddr", "-a", help="NVMe host IP", required=True),
         argument("--trsvcid", "-s", help="Port number", type=int, required=False),
         argument("--adrfam", "-f", help="Address family", default="", choices=get_enum_keys_list(pb2.AddressFamily)),
+        argument("--secure", help="Use secure channel", action='store_true', required=False),
     ]
     listener_del_args = listener_common_args + [
         argument("--host-name", "-t", help="Host name", required=True),
@@ -1033,7 +1036,10 @@ def host_add(self, args):
         out_func, err_func = self.get_output_functions(args)
         if not args.host:
             self.cli.parser.error("--host argument is mandatory for add command")
-        req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=args.host)
+        if args.host == "*" and args.psk:
+            self.cli.parser.error("PSK is only allowed for specific hosts")
+
+        req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=args.host, psk=args.psk)
         try:
             ret = self.stub.add_host(req)
         except Exception as ex:
@@ -1127,16 +1133,17 @@ def host_list(self, args):
             if hosts_info.status == 0:
                 hosts_list = []
                 if hosts_info.allow_any_host:
-                    hosts_list.append(["Any host"])
+                    hosts_list.append(["Any host", "n/a"])
                 for h in hosts_info.hosts:
-                    hosts_list.append([h.nqn])
+                    psk = "Yes" if h.psk else "No"
+                    hosts_list.append([h.nqn, psk])
                 if len(hosts_list) > 0:
                     if args.format == "text":
                         table_format = "fancy_grid"
                     else:
                         table_format = "plain"
                     hosts_out = tabulate(hosts_list,
-                                      headers = [f"Host NQN"],
+                                      headers = ["Host NQN", "PSK"],
                                       tablefmt=table_format, stralign="center")
                     out_func(f"Hosts allowed to access {args.subsystem}:\n{hosts_out}")
                 else:
@@ -1166,6 +1173,7 @@ def host_list(self, args):
     ]
     host_add_args = host_common_args + [
         argument("--host", "-t", help="Host NQN", required=True),
+        argument("--psk", help="Use PSK key", action='store_true', required=False),
     ]
     host_del_args = host_common_args + [
         argument("--host", "-t", help="Host NQN", required=True),
@@ -1204,18 +1212,25 @@ def connection_list(self, args):
             if connections_info.status == 0:
                 connections_list = []
                 for conn in connections_info.connections:
+                    conn_secure = "<n/a>"
+                    conn_psk = "<n/a>"
+                    if conn.connected:
+                        conn_secure = "Yes" if conn.secure else "No"
+                        conn_psk = "Yes" if conn.psk else "No"
                     connections_list.append([conn.nqn,
                                             f"{conn.traddr}:{conn.trsvcid}" if conn.connected else "<n/a>",
                                             "Yes" if conn.connected else "No",
                                             conn.qpairs_count if conn.connected else "<n/a>",
-                                            conn.controller_id if conn.connected else "<n/a>"])
+                                            conn.controller_id if conn.connected else "<n/a>",
+                                            conn_secure,
+                                            conn_psk])
                 if len(connections_list) > 0:
                     if args.format == "text":
                         table_format = "fancy_grid"
                     else:
                         table_format = "plain"
                     connections_out = tabulate(connections_list,
-                                      headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID"],
+                                      headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID", "Secure", "PSK"],
                                       tablefmt=table_format)
                     out_func(f"Connections for {args.subsystem}:\n{connections_out}")
                 else:

control/config.py

@@ -18,6 +18,7 @@ class GatewayConfig:
     """
 
     CEPH_RUN_DIRECTORY = "/var/run/ceph/"
+    CEPH_LIB_DIRECTORY = "/var/lib/ceph/"
 
     def __init__(self, conffile):
         self.filepath = conffile
@@ -52,6 +53,7 @@ def getfloat_with_default(self, section, param, value):
         return self.config.getfloat(section, param, fallback=value)
 
     def dump_config_file(self, logger):
+        # we call this function twice as the tests environment skips the main code, but we still don't want to display twice
         if self.conffile_logged:
             return
 
@@ -70,6 +72,7 @@ def dump_config_file(self, logger):
             pass
 
     def display_environment_info(self, logger):
+        # we call this function twice as the tests environment skips the main code, but we still don't want to display twice
         if self.env_shown:
             return
 

control/discovery.py

@@ -665,7 +665,11 @@ def reply_get_log_page(self, conn, data, cmd_id):
         self_conn = self.conn_vals[conn.fileno()]
         my_omap_dict = self._read_all()
         listeners = self._get_vals(my_omap_dict, GatewayState.LISTENER_PREFIX)
+        secure_listeners = self._get_vals(my_omap_dict, GatewayState.LISTENER_SECURE_PREFIX)
+        listeners += secure_listeners
         hosts = self._get_vals(my_omap_dict, GatewayState.HOST_PREFIX)
+        psk_hosts = self._get_vals(my_omap_dict, GatewayState.HOST_PSK_PREFIX)
+        hosts += psk_hosts
         if len(self_conn.nvmeof_connect_data_hostnqn) != 256:
             self.logger.error("error hostnqn.")
             return -1
@@ -889,7 +893,7 @@ def _state_notify_update(self, update, is_add_req):
 
         should_send_async_event = False
         for key in update.keys():
-            if key.startswith(GatewayState.SUBSYSTEM_PREFIX) or key.startswith(GatewayState.LISTENER_PREFIX):
+            if key.startswith(GatewayState.SUBSYSTEM_PREFIX) or key.startswith(GatewayState.LISTENER_PREFIX) or key.startswith(GatewayState.LISTENER_SECURE_PREFIX):
                 should_send_async_event = True
                 break