Separate namespaces limit to a total and per subsystem one.

Fixes #954

Signed-off-by: Gil Bregman <[email protected]>

ceph-nvmeof.conf

@@ -32,7 +32,8 @@ enable_spdk_discovery_controller = False
 #max_hosts_per_namespace = 1
 #max_namespaces_with_netmask = 1000
 #max_subsystems = 128
-#max_namespaces = 256
+#max_namespaces = 1024
+#max_namespaces_per_subsystem = 256
 #max_hosts_per_subsystem = 32
 
 [gateway-logs]

control/cli.py

@@ -361,6 +361,8 @@ def gw_info(self, args):
                     out_func(f"Gateway's max subsystems: {gw_info.max_subsystems}")
                 if gw_info.max_namespaces:
                     out_func(f"Gateway's max namespaces: {gw_info.max_namespaces}")
+                if gw_info.max_namespaces_per_subsystem:
+                    out_func(f"Gateway's max namespaces per subsystem: {gw_info.max_namespaces_per_subsystem}")
                 if gw_info.max_hosts_per_subsystem:
                     out_func(f"Gateway's max hosts per subsystem: {gw_info.max_hosts_per_subsystem}")
                 if gw_info.spdk_version:

control/grpc.py

@@ -291,7 +291,8 @@ class GatewayService(pb2_grpc.GatewayServicer):
     DHCHAP_CONTROLLER_PREFIX = "dhchap_ctrlr"
     KEYS_DIR = "/var/tmp"
     MAX_SUBSYSTEMS_DEFAULT = 128
-    MAX_NAMESPACES_DEFAULT = 256
+    MAX_NAMESPACES_DEFAULT = 1024
+    MAX_NAMESPACES_PER_SUBSYSTEM_DEFAULT = 256
     MAX_HOSTS_PER_SUBSYS_DEFAULT = 32
 
     def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rpc_lock, omap_lock: OmapLock, group_id: int, spdk_rpc_client, spdk_rpc_subsystems_client, ceph_utils: CephUtils) -> None:
@@ -327,6 +328,7 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp
         self.max_namespaces_with_netmask = self.config.getint_with_default("gateway", "max_namespaces_with_netmask", 1000)
         self.max_subsystems = self.config.getint_with_default("gateway", "max_subsystems", GatewayService.MAX_SUBSYSTEMS_DEFAULT)
         self.max_namespaces = self.config.getint_with_default("gateway", "max_namespaces", GatewayService.MAX_NAMESPACES_DEFAULT)
+        self.max_namespaces_per_subsystem = self.config.getint_with_default("gateway", "max_namespaces_per_subsystem", GatewayService.MAX_NAMESPACES_PER_SUBSYSTEM_DEFAULT)
         self.max_hosts_per_subsystem = self.config.getint_with_default("gateway", "max_hosts_per_subsystem", GatewayService.MAX_HOSTS_PER_SUBSYS_DEFAULT)
         self.gateway_pool = self.config.get_with_default("ceph", "pool", "")
         self.ana_map = defaultdict(dict)
@@ -878,10 +880,12 @@ def create_subsystem_safe(self, request, context):
             return pb2.subsys_status(status = errno.EINVAL, error_message = errmsg, nqn = request.subsystem_nqn)
 
         if not request.max_namespaces:
-            request.max_namespaces = self.max_namespaces
+            request.max_namespaces = self.max_namespaces_per_subsystem
         else:
             if request.max_namespaces > self.max_namespaces:
                 self.logger.warning(f"The requested max number of namespaces for subsystem {request.subsystem_nqn} ({request.max_namespaces}) is greater than the global limit on the number of namespaces ({self.max_namespaces}), will continue")
+            elif request.max_namespaces > self.max_namespaces_per_subsystem:
+                self.logger.warning(f"The requested max number of namespaces for subsystem {request.subsystem_nqn} ({request.max_namespaces}) is greater than the limit on the number of namespaces per subsystem ({self.max_namespaces_per_subsystem}), will continue")
 
         errmsg = ""
         if not GatewayState.is_key_element_valid(request.subsystem_nqn):
@@ -1163,6 +1167,11 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au
         peer_msg = self.get_peer_message(context)
         self.logger.info(f"Received request to add {bdev_name} to {subsystem_nqn} with ANA group id {anagrpid}{nsid_msg}, no_auto_visible: {no_auto_visible}, context: {context}{peer_msg}")
 
+        if subsystem_nqn not in self.subsys_max_ns:
+            errmsg = f"{add_namespace_error_prefix}: No such subsystem"
+            self.logger.error(errmsg)
+            return pb2.nsid_status(status=errno.ENOENT, error_message=errmsg)
+
         if anagrpid > self.subsys_max_ns[subsystem_nqn]:
             errmsg = f"{add_namespace_error_prefix}: Group ID {anagrpid} is bigger than configured maximum {self.subsys_max_ns[subsystem_nqn]}"
             self.logger.error(errmsg)
@@ -1175,23 +1184,28 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au
 
         if no_auto_visible and self.subsystem_nsid_bdev_and_uuid.get_namespace_count(subsystem_nqn,
                                                                                True, 0) >= self.max_namespaces_with_netmask:
-            errmsg = f"Failure adding namespace{nsid_msg} to {subsystem_nqn}: Maximal number of namespaces which are not auto visible ({self.max_namespaces_with_netmask}) has already been reached"
+            errmsg = f"{add_namespace_error_prefix}: Maximal number of namespaces which are not auto visible ({self.max_namespaces_with_netmask}) has already been reached"
             self.logger.error(f"{errmsg}")
             return pb2.req_status(status=errno.E2BIG, error_message=errmsg)
 
         if nsid and nsid > self.subsys_max_ns[subsystem_nqn]:
-            errmsg = f"Failure adding namespace to {subsystem_nqn}: Requested NSID {nsid} is bigger than the maximal one ({self.subsys_max_ns[subsystem_nqn]})"
+            errmsg = f"{add_namespace_error_prefix}: Requested NSID {nsid} is bigger than the maximal one ({self.subsys_max_ns[subsystem_nqn]})"
             self.logger.error(f"{errmsg}")
             return pb2.req_status(status=errno.E2BIG, error_message=errmsg)
 
         if not nsid and self.subsystem_nsid_bdev_and_uuid.get_namespace_count(subsystem_nqn,
                                                                         None, 0) >= self.subsys_max_ns[subsystem_nqn]:
-            errmsg = f"Failure adding namespace to {subsystem_nqn}: Subsystem's maximal number of namespaces ({self.subsys_max_ns[subsystem_nqn]}) has already been reached"
+            errmsg = f"{add_namespace_error_prefix}: Subsystem's maximal number of namespaces ({self.subsys_max_ns[subsystem_nqn]}) has already been reached"
             self.logger.error(f"{errmsg}")
             return pb2.req_status(status=errno.E2BIG, error_message=errmsg)
 
         if self.subsystem_nsid_bdev_and_uuid.get_namespace_count(None, None, 0) >= self.max_namespaces:
-            errmsg = f"Failure adding namespace to {subsystem_nqn}: Maximal number of namespaces ({self.max_namespaces}) has already been reached"
+            errmsg = f"{add_namespace_error_prefix}: Maximal number of namespaces ({self.max_namespaces}) has already been reached"
+            self.logger.error(f"{errmsg}")
+            return pb2.req_status(status=errno.E2BIG, error_message=errmsg)
+
+        if self.subsystem_nsid_bdev_and_uuid.get_namespace_count(subsystem_nqn, None, 0) >= self.subsys_max_ns[subsystem_nqn]:
+            errmsg = f"{add_namespace_error_prefix}: Maximal number of namespaces per subsystem ({self.subsys_max_ns[subsystem_nqn]}) has already been reached"
             self.logger.error(f"{errmsg}")
             return pb2.req_status(status=errno.E2BIG, error_message=errmsg)
 
@@ -2039,7 +2053,7 @@ def namespace_delete_safe(self, request, context):
 
         find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid)
         if find_ret.empty():
-            errmsg = f"Failure deleting namespace: Can't find namespace"
+            errmsg = f"Failure deleting namespace {request.nsid}: Can't find namespace"
             self.logger.error(errmsg)
             return pb2.req_status(status=errno.ENODEV, error_message=errmsg)
         bdev_name = find_ret.bdev
@@ -3696,6 +3710,7 @@ def get_gateway_info_safe(self, request, context):
                                hostname = self.host_name,
                                max_subsystems = self.max_subsystems,
                                max_namespaces = self.max_namespaces,
+                               max_namespaces_per_subsystem = self.max_namespaces_per_subsystem,
                                max_hosts_per_subsystem = self.max_hosts_per_subsystem,
                                status = 0,
                                error_message = os.strerror(0))

control/proto/gateway.proto

@@ -414,6 +414,7 @@ message gateway_info {
 	optional uint32 max_subsystems = 13;
 	optional uint32 max_namespaces = 14;
 	optional uint32 max_hosts_per_subsystem = 15;
+	optional uint32 max_namespaces_per_subsystem = 16;
 }
 
 message cli_version {

tests/ceph-nvmeof.no-huge.conf

@@ -32,7 +32,8 @@ enable_spdk_discovery_controller = False
 #max_hosts_per_namespace = 1
 #max_namespaces_with_netmask = 1000
 #max_subsystems = 128
-#max_namespaces = 256
+#max_namespaces = 1024
+##max_namespaces_per_subsystem = 256
 #max_hosts_per_subsystem = 32
 
 [gateway-logs]

tests/ceph-nvmeof.tls.conf

@@ -30,6 +30,10 @@ enable_spdk_discovery_controller = False
 #spdk_ping_interval_in_seconds = 2.0
 #max_hosts_per_namespace = 1
 #max_namespaces_with_netmask = 1000
+#max_subsystems = 128
+#max_namespaces = 1024
+#max_namespaces_per_subsystem = 256
+#max_hosts_per_subsystem = 32
 
 [gateway-logs]
 log_level=debug

tests/test_cli.py

@@ -22,6 +22,8 @@
 image9 = "mytestdevimage9"
 image10 = "mytestdevimage10"
 image11 = "mytestdevimage11"
+image12 = "mytestdevimage12"
+image13 = "mytestdevimage13"
 pool = "rbd"
 subsystem = "nqn.2016-06.io.spdk:cnode1"
 subsystem2 = "nqn.2016-06.io.spdk:cnode2"
@@ -69,7 +71,8 @@ def gateway(config):
     config.config["gateway"]["group"] = group_name
     config.config["gateway"]["max_namespaces_with_netmask"] = "3"
     config.config["gateway"]["max_subsystems"] = "3"
-    config.config["gateway"]["max_namespaces"] = "11"
+    config.config["gateway"]["max_namespaces"] = "12"
+    config.config["gateway"]["max_namespaces_per_subsystem"] = "11"
     config.config["gateway"]["max_hosts_per_subsystem"] = "4"
     config.config["gateway-logs"]["log_level"] = "debug"
     ceph_utils = CephUtils(config)
@@ -154,7 +157,8 @@ def test_get_gateway_info(self, caplog, gateway):
         assert gw_info.name == gw.gateway_name
         assert gw_info.hostname == gw.host_name
         assert gw_info.max_subsystems == 3
-        assert gw_info.max_namespaces == 11
+        assert gw_info.max_namespaces == 12
+        assert gw_info.max_namespaces_per_subsystem == 11
         assert gw_info.max_hosts_per_subsystem == 4
         assert gw_info.status == 0
         assert gw_info.bool_status == True
@@ -201,7 +205,7 @@ def test_create_subsystem(self, caplog, gateway):
         assert f"contains invalid characters" in caplog.text
         caplog.clear()
         cli(["subsystem", "add", "--subsystem", subsystem, "--max-namespaces", "2049", "--no-group-append"])
-        assert f"The requested max number of namespaces for subsystem {subsystem} (2049) is greater than the global limit on the number of namespaces (11), will continue" in caplog.text
+        assert f"The requested max number of namespaces for subsystem {subsystem} (2049) is greater than the global limit on the number of namespaces (12), will continue" in caplog.text
         assert f"Adding subsystem {subsystem}: Successful" in caplog.text
         cli(["--format", "json", "subsystem", "list"])
         assert f'"serial_number": "{serial}"' not in caplog.text
@@ -490,10 +494,15 @@ def test_add_all_hosts_to_namespace(self, caplog, gateway):
         cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "8", "--host-nqn", "*"])
         assert f"Failure adding host to namespace 8 on {subsystem}, host can't be \"*\"" in caplog.text
 
+    def test_add_namespace_no_such_subsys(self, caplog, gateway):
+        caplog.clear()
+        cli(["namespace", "add", "--subsystem", f"{subsystem3}", "--rbd-pool", pool, "--rbd-image", image13, "--size", "16MB", "--rbd-create-image"])
+        assert f"Failure adding namespace to {subsystem3}: No such subsystem"
+
     def test_add_too_many_namespaces_to_a_subsystem(self, caplog, gateway):
         caplog.clear()
         cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image9, "--nsid", "3000", "--size", "16MB", "--rbd-create-image"])
-        assert f"Failure adding namespace to {subsystem}: Requested NSID 3000 is bigger than the maximal one" in caplog.text
+        assert f"Failure adding namespace using NSID 3000 to {subsystem}: Requested NSID 3000 is bigger than the maximal one (2049)" in caplog.text
         assert f"Received request to delete bdev" in caplog.text
         caplog.clear()
         cli(["subsystem", "add", "--subsystem", subsystem5, "--no-group-append", "--max-namespaces", "1"])
@@ -554,7 +563,10 @@ def test_list_namespace_with_no_hosts(self, caplog, gateway):
     def test_add_too_many_namespaces(self, caplog, gateway):
         caplog.clear()
         cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image11, "--size", "16MB", "--rbd-create-image"])
-        assert f"Failure adding namespace to {subsystem}: Maximal number of namespaces (11) has already been reached" in caplog.text
+        assert f"Adding namespace 12 to {subsystem}: Successful" in caplog.text
+        caplog.clear()
+        cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image12, "--size", "16MB", "--rbd-create-image"])
+        assert f"Failure adding namespace to {subsystem}: Maximal number of namespaces (12) has already been reached" in caplog.text
 
     def test_resize_namespace(self, caplog, gateway):
         gw, stub = gateway
@@ -647,8 +659,8 @@ def test_resize_namespace(self, caplog, gateway):
         assert '"nsid": 4' not in caplog.text
         assert '"nsid": 5' not in caplog.text
         caplog.clear()
-        cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "12", "--size", "128MB"])
-        assert f"Failure resizing namespace 12 on {subsystem}: Can't find namespace" in caplog.text
+        cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "22", "--size", "128MB"])
+        assert f"Failure resizing namespace 22 on {subsystem}: Can't find namespace" in caplog.text
         caplog.clear()
         cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "32MB"])
         assert f"Failure resizing namespace 6 on {subsystem}: new size 33554432 bytes is smaller than current size 67108864 bytes" in caplog.text
@@ -1111,7 +1123,8 @@ def test_create_subsys_group_name(self, caplog, gateway):
 class TestTooManySubsystemsAndHosts:
     def test_add_too_many_subsystem(self, caplog, gateway):
         caplog.clear()
-        cli(["subsystem", "add", "--subsystem", subsystem6, "--no-group-append"])
+        cli(["subsystem", "add", "--subsystem", subsystem6, "--no-group-append", "--max-namespaces", "12"])
+        assert f"The requested max number of namespaces for subsystem {subsystem6} (12) is greater than the limit on the number of namespaces per subsystem (11), will continue" in caplog.text
         assert f"Adding subsystem {subsystem6}: Successful" in caplog.text
         caplog.clear()
         cli(["subsystem", "add", "--subsystem", subsystem7, "--no-group-append"])