diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 6ea63346c..ef1d0f318 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -223,13 +223,29 @@ jobs: run: | make demo OPTS=-T NVMEOF_CONTAINER_NAME="ceph-nvmeof_nvmeof_1" - - name: Get subsystems + - name: List resources run: | # https://github.com/actions/toolkit/issues/766 shopt -s expand_aliases eval $(make alias) - nvmeof-cli get_subsystems - nvmeof-cli-ipv6 get_subsystems + cephnvmf subsystem list + subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//') + for sub in $subs + do + cephnvmf namespace list --subsystem $sub + cephnvmf listener list --subsystem $sub + cephnvmf host list --subsystem $sub + done + + - name: Verify no HA + run: | + shopt -s expand_aliases + eval $(make alias) + ha_status=$(cephnvmf --output stdio --format json subsystem list | grep "enable_ha" | grep "true" | tr -d '\n\r') + if [ -n "$ha_status" ]; then + echo "Found a subsystem with HA enabled, failing" + exit 1 + fi - name: Run bdevperf run: | @@ -390,21 +406,20 @@ jobs: gw1=$(container_ip $GW1) echo ℹ️ Using GW RPC $GW1 address $gw1 port $NVMEOF_GW_PORT - cli_gw $gw1 get_subsystems - cli_gw $gw1 create_bdev --pool $RBD_POOL --image $RBD_IMAGE_NAME --bdev $BDEV_NAME - cli_gw $gw1 create_subsystem --subnqn $NQN --serial $SERIAL - cli_gw $gw1 add_namespace --subnqn $NQN --bdev $BDEV_NAME + cli_gw $gw1 subsystem list + cli_gw $gw1 subsystem add --subsystem $NQN --serial $SERIAL + cli_gw $gw1 namespace add --subsystem $NQN --rbd-pool $RBD_POOL --rbd-image $RBD_IMAGE_NAME for gw in $GW1 $GW2; do ip=$(container_ip $gw) name=$(container_id $gw) # default hostname - container id echo ℹ️ Create listener address $ip gateway $name - cli_gw $ip create_listener --subnqn $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT + cli_gw $ip listener add --subsystem $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT done - cli_gw $gw1 add_host --subnqn $NQN --host "*" + cli_gw $gw1 host add --subsystem $NQN --host "*" for gw in $GW1 $GW2; do ip=$(container_ip $gw) echo ℹ️ Subsystems for name $gw ip $ip - cli_gw $ip get_subsystems + cli_gw $ip subsystem list done - name: Run bdevperf discovery diff --git a/Dockerfile b/Dockerfile index c001fc4ee..ed3892ac8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,10 @@ FROM quay.io/ceph/spdk:${NVMEOF_SPDK_VERSION:-NULL} AS base-gateway RUN \ --mount=type=cache,target=/var/cache/dnf \ --mount=type=cache,target=/var/lib/dnf \ - dnf install -y python3-rados + dnf install -y python3-rados python3-pip +RUN \ + --mount=type=cache,target=/root/.cache/pip \ + pip install -U pip setuptools ENTRYPOINT ["python3", "-m", "control"] CMD ["-c", "/src/ceph-nvmeof.conf"] @@ -99,6 +102,17 @@ LABEL io.ceph.component="$NVMEOF_NAME" \ ENV PYTHONPATH=$APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib +RUN \ + --mount=type=cache,target=/var/cache/dnf \ + --mount=type=cache,target=/var/lib/dnf \ + dnf install -y python3-pip +RUN \ + --mount=type=cache,target=/root/.cache/pip \ + TABULATE_INSTALL=lib-only pip install -U --target $APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib tabulate +RUN \ + --mount=type=cache,target=/root/.cache/pip \ + pip install -U --target $APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib pyyaml + WORKDIR $APPDIR #------------------------------------------------------------------------------ diff --git a/README.md b/README.md index 10758d25c..319977c33 100644 --- a/README.md +++ b/README.md @@ -92,36 +92,25 @@ The following command executes all the steps required to set up the NVMe-oF envi ```bash $ make demo - -DOCKER_BUILDKIT=1 docker-compose exec ceph-vstart-cluster bash -c "rbd info demo_image || rbd create demo_image --size 10M" +docker-compose exec ceph bash -c "rbd -p rbd info demo_image || rbd -p rbd create demo_image --size 10M" rbd: error opening image demo_image: (2) No such file or directory - -DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_bdev --pool rbd --image demo_image --bdev demo_bdev -Creating nvmeof_ceph-nvmeof-cli_run ... done -INFO:__main__:Created bdev demo_bdev: True - -DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_subsystem --subnqn nqn.2016-06.io.spdk:cnode1 --serial SPDK00000000000001 -Creating nvmeof_ceph-nvmeof-cli_run ... done -INFO:__main__:Created subsystem nqn.2016-06.io.spdk:cnode1: True - -DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 add_namespace --subnqn nqn.2016-06.io.spdk:cnode1 --bdev demo_bdev -Creating nvmeof_ceph-nvmeof-cli_run ... done -INFO:__main__:Added namespace 1 to nqn.2016-06.io.spdk:cnode1: True - -DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_listener --subnqn nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420 -Creating nvmeof_ceph-nvmeof-cli_run ... done -INFO:__main__:Created nqn.2016-06.io.spdk:cnode1 listener: True - -DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 add_host --subnqn nqn.2016-06.io.spdk:cnode1 --host "*" -Creating nvmeof_ceph-nvmeof-cli_run ... done -INFO:__main__:Allowed open host access to nqn.2016-06.io.spdk:cnode1: True +docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 subsystem add --subsystem "nqn.2016-06.io.spdk:cnode1" --ana-reporting --enable-ha +Adding subsystem nqn.2016-06.io.spdk:cnode1: Successful +docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 namespace add --subsystem "nqn.2016-06.io.spdk:cnode1" --rbd-pool rbd --rbd-image demo_image +Adding namespace 1 to nqn.2016-06.io.spdk:cnode1, load balancing group 1: Successful +docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 listener add --subsystem "nqn.2016-06.io.spdk:cnode1" --gateway-name fbca1a3d3ed8 --traddr 192.168.13.3 --trsvcid 4420 +Adding listener 192.168.13.3:4420 to nqn.2016-06.io.spdk:cnode1: Successful +docker-compose run --rm nvmeof-cli --server-address 2001:db8::3 --server-port 5500 listener add --subsystem "nqn.2016-06.io.spdk:cnode1" --gateway-name fbca1a3d3ed8 --traddr 2001:db8::3 --trsvcid 4420 --adrfam IPV6 +Adding listener [2001:db8::3]:4420 to nqn.2016-06.io.spdk:cnode1: Successful +docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 host add --subsystem "nqn.2016-06.io.spdk:cnode1" --host "*" +Allowing any host for nqn.2016-06.io.spdk:cnode1: Successful ``` #### Manual Steps The same configuration can also be manually run: -1. First of all, let's create the `nvmeof-cli` shortcut to interact with the NVMe-oF gateway: +1. First of all, let's create the `cephnvmf` shortcut to interact with the NVMe-oF gateway: ```bash eval $(make alias) @@ -133,34 +122,28 @@ The same configuration can also be manually run: make rbd ``` -1. Create a bdev (Block Device) from an RBD image: - - ```bash - nvmeof-cli create_bdev --pool rbd --image demo_image --bdev demo_bdev - ``` - 1. Create a subsystem: ```bash - nvmeof-cli create_subsystem --subnqn nqn.2016-06.io.spdk:cnode1 --serial SPDK00000000000001 + cephnvmf subsystem add --subsystem nqn.2016-06.io.spdk:cnode1 ``` 1. Add a namespace: ```bash - nvmeof-cli add_namespace --subnqn nqn.2016-06.io.spdk:cnode1 --bdev demo_bdev + cephnvmf namespace add --subsystem nqn.2016-06.io.spdk:cnode1 --rbd-pool rbd --rbd-image demo_image ``` 1. Create a listener so that NVMe initiators can connect to: ```bash - nvmeof-cli create_listener ---subnqn nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420 + cephnvmf listener add ---subsystem nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420 ``` 1. Define which hosts can connect: ```bash - nvmeof-cli add_host --subnqn nqn.2016-06.io.spdk:cnode1 --host "*" + cephnvmf host add --subsystem nqn.2016-06.io.spdk:cnode1 --host "*" ``` diff --git a/control/cli.py b/control/cli.py index 465ac0193..2ac3066c5 100644 --- a/control/cli.py +++ b/control/cli.py @@ -12,20 +12,55 @@ import json import logging import sys +import errno import os +import yaml from functools import wraps from google.protobuf import json_format +from tabulate import tabulate from .proto import gateway_pb2_grpc as pb2_grpc from .proto import gateway_pb2 as pb2 from .config import GatewayConfig +from .config import GatewayEnumUtils + +def errprint(msg): + print(msg, file = sys.stderr) def argument(*name_or_flags, **kwargs): """Helper function to format arguments for argparse command decorator.""" - return (list(name_or_flags), kwargs) +def get_enum_keys_list(e_type, include_first = False): + k_list = [] + for k in e_type.keys(): + k_list.append(k.lower()) + k_list.append(k.upper()) + if not include_first: + k_list = k_list[1:] + + return k_list + +class ErrorCatchingArgumentParser(argparse.ArgumentParser): + def __init__(self, *args, **kwargs): + self.logger = logging.getLogger(__name__) + super(ErrorCatchingArgumentParser, self).__init__(*args, **kwargs) + + def exit(self, status = 0, message = None): + if status != 0: + if message: + self.logger.error(message) + else: + if message: + self.logger.info(message) + exit(status) + + def error(self, message): + self.print_usage() + if message: + self.logger.error(f"error: {message}") + exit(2) class Parser: """Class to simplify creation of client CLI. @@ -36,17 +71,33 @@ class Parser: """ def __init__(self): - self.parser = argparse.ArgumentParser( + self.parser = ErrorCatchingArgumentParser( prog="python3 -m control.cli", description="CLI to manage NVMe gateways") + self.parser.add_argument( + "--format", + help="CLI output format", + type=str, + default="text", + choices=["text", "json", "yaml"], + required=False) + self.parser.add_argument( + "--output", + help="CLI output method", + type=str, + default="log", + choices=["log", "stdio"], + required=False) self.parser.add_argument( "--server-address", + "--ip", default="localhost", type=str, - help="Server address", + help="Server IP address", ) self.parser.add_argument( "--server-port", + "--port", default=5500, type=int, help="Server port", @@ -113,7 +164,7 @@ class GatewayClient: def __init__(self): self._stub = None - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(format='%(message)s', level=logging.DEBUG) self.logger = logging.getLogger(__name__) @property @@ -156,152 +207,489 @@ def connect(self, host, port, client_key, client_cert, server_cert): # Bind the client and the server self._stub = pb2_grpc.GatewayStub(channel) - @cli.cmd([ - argument("-i", "--image", help="RBD image name", required=True), - argument("-p", "--pool", help="RBD pool name", required=True), - argument("-b", "--bdev", help="Bdev name"), - argument("-s", - "--block-size", - help="Block size", - type=int, - default=512), - ]) - def create_bdev(self, args): - """Creates a bdev from an RBD image.""" - req = pb2.create_bdev_req( - rbd_pool_name=args.pool, - rbd_image_name=args.image, - block_size=args.block_size, - bdev_name=args.bdev, - ) - ret = self.stub.create_bdev(req) - self.logger.info(f"Created bdev {ret.bdev_name}: {ret.status}") + def format_adrfam(self, adrfam): + adrfam = adrfam.upper() + if adrfam == "IPV4": + adrfam = "IPv4" + elif adrfam == "IPV6": + adrfam = "IPv6" + + return adrfam + + def get_output_functions(self, args): + if args.output == "log": + return (self.logger.info, self.logger.error) + elif args.output == "stdio": + return (print, errprint) + else: + self.cli.parser.error("invalid --output value") @cli.cmd([ - argument("-b", "--bdev", help="Bdev name", required=True), - argument("-s", "--size", help="New size in MiB", type=int, required=True), ]) - def resize_bdev(self, args): - """Resizes a bdev.""" - req = pb2.resize_bdev_req( - bdev_name=args.bdev, - new_size=args.size, - ) - ret = self.stub.resize_bdev(req) - self.logger.info(f"Resized bdev {args.bdev}: {ret.status}") + def version(self, args): + """Get CLI version""" + rc = 0 + out_func, err_func = self.get_output_functions(args) + ver = os.getenv("NVMEOF_VERSION") + if args.format == "text": + if not ver: + err_func("Can't get CLI version") + rc = errno.ENOKEY + else: + out_func(f"CLI version: {ver}") + rc = 0 + elif args.format == "json" or args.format == "yaml": + if not ver: + rc = errno.ENOKEY + errmsg = "Can't get CLI version" + else: + rc = 0 + errmsg = os.strerror(rc) + cli_ver = pb2.cli_version(status=rc, error_message=errmsg, version=ver) + out_ver = json_format.MessageToJson(cli_ver, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{out_ver}") + else: + obj = json.loads(out_ver) + out_func(yaml.dump(obj)) + else: + assert False + + return rc + + def gw_get_info(self): + ver = os.getenv("NVMEOF_VERSION") + req = pb2.get_gateway_info_req(cli_version=ver) + gw_info = self.stub.get_gateway_info(req) + return gw_info + + def gw_info(self, args): + """Get gateway's information""" + + out_func, err_func = self.get_output_functions(args) + try: + gw_info = self.gw_get_info() + except Exception as ex: + gw_info = pb2.gateway_info(status = errno.EINVAL, error_message = f"Failure getting gateway's information:\n{ex}") + + if args.format == "text": + if gw_info.status == 0: + if gw_info.version: + out_func(f"Gateway's version: {gw_info.version}") + if gw_info.name: + out_func(f"Gateway's name: {gw_info.name}") + if gw_info.group: + out_func(f"Gateway's group: {gw_info.group}") + out_func(f"Gateway's address: {gw_info.addr}") + out_func(f"Gateway's port: {gw_info.port}") + else: + err_func(f"{gw_info.error_message}") + elif args.format == "json" or args.format == "yaml": + gw_info_str = json_format.MessageToJson( + gw_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{gw_info_str}") + else: + obj = json.loads(gw_info_str) + out_func(yaml.dump(obj)) + else: + assert False + + return gw_info.status + + def gw_version(self, args): + """Get gateway's version""" + + out_func, err_func = self.get_output_functions(args) + try: + gw_info = self.gw_get_info() + except Exception as ex: + gw_info = pb2.gateway_info(status = errno.EINVAL, error_message = f"Failure getting gateway's version:\n{ex}") + + if args.format == "text": + if gw_info.status == 0: + out_func(f"Gateway's version: {gw_info.version}") + else: + err_func(f"{gw_info.error_message}") + elif args.format == "json" or args.format == "yaml": + gw_ver = pb2.gw_version(status=gw_info.status, error_message=gw_info.error_message, version=gw_info.version) + out_ver = json_format.MessageToJson(gw_ver, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{out_ver}") + else: + obj = json.loads(out_ver) + out_func(yaml.dump(obj)) + else: + assert False + + return gw_info.status @cli.cmd([ - argument("-b", "--bdev", help="Bdev name", required=True), - argument("-f", "--force", help="Delete any namespace using this bdev before deleting bdev", action='store_true', required=False), + argument("gw_command", help="gw sub-command", choices=["version", "info"]), ]) - def delete_bdev(self, args): - """Deletes a bdev.""" - req = pb2.delete_bdev_req(bdev_name=args.bdev, force=args.force) - ret = self.stub.delete_bdev(req) - self.logger.info(f"Deleted bdev {args.bdev}: {ret.status}") + def gw(self, args): + """Gateway commands""" + + if args.gw_command == "info": + return self.gw_info(args) + elif args.gw_command == "version": + return self.gw_version(args) + assert False + + def log_level_disable(self, args): + """Disable SPDK nvmf log flags""" + + out_func, err_func = self.get_output_functions(args) + if args.level != None: + self.cli.parser.error("--level argument is not allowed for disable command") + if args.print != None: + self.cli.parser.error("--print argument is not allowed for disable command") + + req = pb2.disable_spdk_nvmf_logs_req() + try: + ret = self.stub.disable_spdk_nvmf_logs(req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure disabling SPDK nvmf log flags:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Disable SPDK nvmf log flags: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def log_level_get(self, args): + """Get SPDK log levels and nvmf log flags""" + + out_func, err_func = self.get_output_functions(args) + if args.level != None: + self.cli.parser.error("--level argument is not allowed for get command") + if args.print != None: + self.cli.parser.error("--print argument is not allowed for get command") + + req = pb2.get_spdk_nvmf_log_flags_and_level_req() + try: + ret = self.stub.get_spdk_nvmf_log_flags_and_level(req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure getting SPDK log levels and nvmf log flags:\n{ex}") + + if args.format == "text": + if ret.status == 0: + for flag in ret.nvmf_log_flags: + enabled_str = "enabled" if flag.enabled else "disabled" + out_func(f"SPDK nvmf log flag \"{flag.name}\" is {enabled_str}") + level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, ret.log_level) + out_func(f"SPDK log level is {level}") + level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, ret.log_print_level) + out_func(f"SPDK log print level is {level}") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + out_log_level = json_format.MessageToJson(ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{out_log_level}") + else: + obj = json.loads(out_log_level) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def log_level_set(self, args): + """Set SPDK log levels and nvmf log flags""" + rc = 0 + errmsg = "" + + out_func, err_func = self.get_output_functions(args) + log_level = None + print_level = None + + if args.level: + log_level = args.level.upper() + + if args.print: + print_level = args.print.upper() + + try: + req = pb2.set_spdk_nvmf_logs_req(log_level=log_level, print_level=print_level) + except ValueError as err: + errstr = str(err) + if errstr.startswith("unknown enum label "): + errstr = errstr.removeprefix("unknown enum label ") + self.cli.parser.error(f"invalid log level {errstr}") + else: + self.cli.parser.error(f"{err}") + + try: + ret = self.stub.set_spdk_nvmf_logs(req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure setting SPDK log levels and nvmf log flags:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Set SPDK log levels and nvmf log flags: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-s", "--serial", help="Serial number", required=False), - argument("-m", "--max-namespaces", help="Maximum number of namespaces", type=int, default=0, required=False), - argument("-a", "--ana-reporting", help="Enable ANA reporting", action='store_true', required=False), - argument("-t", "--enable-ha", help="Enable automatic HA", action='store_true', required=False), + argument("log_level_command", help="log level sub-command", choices=["get", "set", "disable"]), + argument("--level", "-l", help="SPDK nvmf log level", required=False, + type=str, choices=["ERROR", "WARNING", "NOTICE", "INFO", "DEBUG", + "error", "warning", "notice", "info", "debug"]), + argument("--print", "-p", help="SPDK nvmf log print level", required=False, + type=str, choices=["ERROR", "WARNING", "NOTICE", "INFO", "DEBUG", + "error", "warning", "notice", "info", "debug"]), ]) - def create_subsystem(self, args): - """Creates a subsystem.""" - req = pb2.create_subsystem_req(subsystem_nqn=args.subnqn, - serial_number=args.serial, + def log_level(self, args): + """Log level commands""" + if args.log_level_command == "get": + return self.log_level_get(args) + elif args.log_level_command == "set": + return self.log_level_set(args) + elif args.log_level_command == "disable": + return self.log_level_disable(args) + assert False + + def subsystem_add(self, args): + """Create a subsystem""" + + out_func, err_func = self.get_output_functions(args) + if args.max_namespaces == None: + args.max_namespaces = 256 + if args.max_namespaces < 0: + self.cli.parser.error("--max-namespaces value must be positive") + if not args.subsystem: + self.cli.parser.error("--subsystem argument is mandatory for add command") + if args.force: + self.cli.parser.error("--force argument is not allowed for add command") + if args.enable_ha and not args.ana_reporting: + self.cli.parser.error("ANA reporting must be enabled when HA is active") + if args.subsystem == GatewayConfig.DISCOVERY_NQN: + self.cli.parser.error("Can't add a discovery subsystem") + + req = pb2.create_subsystem_req(subsystem_nqn=args.subsystem, + serial_number=args.serial_number, max_namespaces=args.max_namespaces, ana_reporting=args.ana_reporting, enable_ha=args.enable_ha) - ret = self.stub.create_subsystem(req) - self.logger.info(f"Created subsystem {args.subnqn}: {ret.status}") + try: + ret = self.stub.create_subsystem(req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure adding subsystem {args.subsystem}:\n{ex}") - @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - ]) - def delete_subsystem(self, args): - """Deletes a subsystem.""" - req = pb2.delete_subsystem_req(subsystem_nqn=args.subnqn) - ret = self.stub.delete_subsystem(req) - self.logger.info(f"Deleted subsystem {args.subnqn}: {ret.status}") + if args.format == "text": + if ret.status == 0: + out_func(f"Adding subsystem {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False - @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-b", "--bdev", help="Bdev name", required=True), - argument("-i", "--nsid", help="Namespace ID", type=int), - argument("-a", "--anagrpid", help="ANA group ID", type=int), - ]) - def add_namespace(self, args): - """Adds a namespace to a subsystem.""" - if args.anagrpid == 0: - args.anagrpid = 1 + return ret.status - req = pb2.add_namespace_req(subsystem_nqn=args.subnqn, - bdev_name=args.bdev, - nsid=args.nsid, - anagrpid=args.anagrpid) - ret = self.stub.add_namespace(req) - self.logger.info( - f"Added namespace {ret.nsid} to {args.subnqn}, ANA group id {args.anagrpid} : {ret.status}") + def subsystem_del(self, args): + """Delete a subsystem""" - @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-i", "--nsid", help="Namespace ID", type=int, required=True), - ]) - def remove_namespace(self, args): - """Removes a namespace from a subsystem.""" - req = pb2.remove_namespace_req(subsystem_nqn=args.subnqn, - nsid=args.nsid) - ret = self.stub.remove_namespace(req) - self.logger.info( - f"Removed namespace {args.nsid} from {args.subnqn}:" - f" {ret.status}") + out_func, err_func = self.get_output_functions(args) + if not args.subsystem: + self.cli.parser.error("--subsystem argument is mandatory for del command") + if args.serial_number != None: + self.cli.parser.error("--serial-number argument is not allowed for del command") + if args.max_namespaces != None: + self.cli.parser.error("--max-namespaces argument is not allowed for del command") + if args.ana_reporting: + self.cli.parser.error("--ana-reporting argument is not allowed for del command") + if args.enable_ha: + self.cli.parser.error("--enable-ha argument is not allowed for del command") + if args.subsystem == GatewayConfig.DISCOVERY_NQN: + self.cli.parser.error("Can't delete a discovery subsystem") - @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-t", "--host", help="Host NQN", required=True), - ]) - def add_host(self, args): - """Adds a host to a subsystem.""" - req = pb2.add_host_req(subsystem_nqn=args.subnqn, - host_nqn=args.host) - ret = self.stub.add_host(req) - if args.host == "*": - self.logger.info( - f"Allowed open host access to {args.subnqn}: {ret.status}") + req = pb2.delete_subsystem_req(subsystem_nqn=args.subsystem, force=args.force) + try: + ret = self.stub.delete_subsystem(req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure deleting subsystem {args.subsystem}:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Deleting subsystem {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) else: - self.logger.info( - f"Added host {args.host} access to {args.subnqn}:" - f" {ret.status}") + assert False - @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-t", "--host", help="Host NQN", required=True), - ]) - def remove_host(self, args): - """Removes a host from a subsystem.""" - req = pb2.remove_host_req(subsystem_nqn=args.subnqn, - host_nqn=args.host) - ret = self.stub.remove_host(req) - if args.host == "*": - self.logger.info( - f"Disabled open host access to {args.subnqn}: {ret.status}") + return ret.status + + def subsystem_list(self, args): + """List subsystems""" + + out_func, err_func = self.get_output_functions(args) + if args.max_namespaces != None: + self.cli.parser.error("--max-namespaces argument is not allowed for list command") + if args.ana_reporting: + self.cli.parser.error("--ana-reporting argument is not allowed for list command") + if args.enable_ha: + self.cli.parser.error("--enable-ha argument is not allowed for list command") + if args.force: + self.cli.parser.error("--force argument is not allowed for list command") + + subsystems = None + try: + subsystems = self.stub.list_subsystems(pb2.list_subsystems_req(subsystem_nqn=args.subsystem, serial_number=args.serial_number)) + except Exception as ex: + subsystems = pb2.subsystems_info(status = errno.EINVAL, error_message = f"Failure listing subsystems:\n{ex}") + + if args.format == "text": + if subsystems.status == 0: + subsys_list = [] + for s in subsystems.subsystems: + if args.subsystem and args.subsystem != s.nqn: + err_func("Failure listing subsystem {args.subsystem}: Got subsystem {s.nqn} instead") + return errno.ENODEV + if args.serial_number and args.serial_number != s.serial_number: + err_func("Failure listing subsystem with serial number {args.serial_number}: Got serial number {s.serial_number} instead") + return errno.ENODEV + ctrls_id = f"{s.min_cntlid}-{s.max_cntlid}" + ha_str = "enabled" if s.enable_ha else "disabled" + one_subsys = [s.subtype, s.nqn, ha_str, s.serial_number, s.model_number, ctrls_id, s.namespace_count] + subsys_list.append(one_subsys) + if len(subsys_list) > 0: + subsys_out = tabulate(subsys_list, + headers = ["Subtype", "NQN", "HA State", "Serial Number", "Model Number", "Controller IDs", + "Namespace\nCount"], + tablefmt="fancy_grid") + prefix = "Subsystems" + if args.subsystem: + prefix = f"Subsystem {args.subsystem}" + if args.serial_number: + prefix = prefix + f" with serial number {args.serial_number}" + out_func(f"{prefix}:\n{subsys_out}") + else: + if args.subsystem: + out_func(f"No subsystem {args.subsystem}") + elif args.serial_number: + out_func(f"No subsystem with serial number {args.serial_number}") + else: + out_func(f"No subsystems") + else: + err_func(f"{subsystems.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + subsystems, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) else: - self.logger.info( - f"Removed host {args.host} access from {args.subnqn}:" - f" {ret.status}") + assert False + + return subsystems.status @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-g", "--gateway-name", help="Gateway name", required=True), - argument("-t", "--trtype", help="Transport type", default="TCP"), - argument("-f", "--adrfam", help="Address family", default="IPV4"), - argument("-a", "--traddr", help="NVMe host IP", required=True), - argument("-s", "--trsvcid", help="Port number", default="4420", required=False), + argument("subsystem_command", help="subsystem sub-command", choices=["add", "del", "delete", "remove", "list"]), + argument("--subsystem", "-n", help="Subsystem NQN", required=False), + argument("--serial-number", "-s", help="Serial number", required=False), + argument("--max-namespaces", "-m", help="Maximum number of namespaces", type=int, required=False), + argument("--ana-reporting", "-a", help="Enable ANA reporting", action='store_true', required=False), + argument("--enable-ha", "-t", help="Enable automatic HA", action='store_true', required=False), + argument("--force", help="Delete subsytem's namespaces if any, then delete subsystem. If not set a subsystem deletion would fail in case it contains namespaces", action='store_true', required=False), ]) - def create_listener(self, args): - """Creates a listener for a subsystem at a given IP/Port.""" + def subsystem(self, args): + """Subsystems commands""" + if args.subsystem_command == "add": + return self.subsystem_add(args) + elif args.subsystem_command == "del" or args.subsystem_command == "delete" or args.subsystem_command == "remove": + return self.subsystem_del(args) + elif args.subsystem_command == "list": + return self.subsystem_list(args) + assert False + + def listener_add(self, args): + """Create a listener""" + + out_func, err_func = self.get_output_functions(args) + if not args.gateway_name: + self.cli.parser.error("--gateway-name argument is mandatory for add command") + if not args.traddr: + self.cli.parser.error("--traddr argument is mandatory for add command") + + if not args.trsvcid: + args.trsvcid = 4420 + elif args.trsvcid < 0: + self.cli.parser.error("trsvcid value must be positive") + if not args.trtype: + args.trtype = "TCP" + if not args.adrfam: + args.adrfam = "IPV4" + traddr = GatewayConfig.escape_address_if_ipv6(args.traddr) trtype = None adrfam = None @@ -309,36 +697,62 @@ def create_listener(self, args): trtype = args.trtype.upper() if args.adrfam: adrfam = args.adrfam.lower() + + req = pb2.create_listener_req( + nqn=args.subsystem, + gateway_name=args.gateway_name, + trtype=trtype, + adrfam=adrfam, + traddr=traddr, + trsvcid=args.trsvcid, + auto_ha_state="AUTO_HA_UNSET", + ) + try: - req = pb2.create_listener_req( - nqn=args.subnqn, - gateway_name=args.gateway_name, - trtype=trtype, - adrfam=adrfam, - traddr=traddr, - trsvcid=args.trsvcid, - auto_ha_state="AUTO_HA_UNSET", - ) ret = self.stub.create_listener(req) - self.logger.info(f"Created {args.subnqn} listener at {traddr}:{args.trsvcid}: {ret.status}") - except ValueError as err: - self.logger.error(f"{err}") - self.logger.info(f"Created {args.subnqn} listener at {traddr}:{args.trsvcid}: {False}") - raise except Exception as ex: - self.logger.info(f"Created {args.subnqn} listener at {traddr}:{args.trsvcid}: {False}") - raise + ret = pb2.req_status(status = errno.EINVAL, + error_message = f"Failure adding {args.subsystem} listener at {traddr}:{args.trsvcid}:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Adding {args.subsystem} listener at {traddr}:{args.trsvcid}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def listener_del(self, args): + """Delete a listener""" + + out_func, err_func = self.get_output_functions(args) + if not args.gateway_name: + self.cli.parser.error("--gateway-name argument is mandatory for del command") + if not args.traddr: + self.cli.parser.error("--traddr argument is mandatory for del command") + + if not args.trsvcid: + args.trsvcid = 4420 + elif args.trsvcid < 0: + self.cli.parser.error("trsvcid value must be positive") + if not args.trtype: + args.trtype = "TCP" + if not args.adrfam: + args.adrfam = "IPV4" - @cli.cmd([ - argument("-n", "--subnqn", help="Subsystem NQN", required=True), - argument("-g", "--gateway-name", help="Gateway name", required=True), - argument("-t", "--trtype", help="Transport type", default="TCP"), - argument("-f", "--adrfam", help="Address family", default="IPV4"), - argument("-a", "--traddr", help="NVMe host IP", required=True), - argument("-s", "--trsvcid", help="Port number", default="4420", required=False), - ]) - def delete_listener(self, args): - """Deletes a listener from a subsystem at a given IP/Port.""" traddr = GatewayConfig.escape_address_if_ipv6(args.traddr) trtype = None adrfam = None @@ -346,99 +760,877 @@ def delete_listener(self, args): trtype = args.trtype.upper() if args.adrfam: adrfam = args.adrfam.lower() + + req = pb2.delete_listener_req( + nqn=args.subsystem, + gateway_name=args.gateway_name, + trtype=trtype, + adrfam=adrfam, + traddr=traddr, + trsvcid=args.trsvcid, + ) + try: - req = pb2.delete_listener_req( - nqn=args.subnqn, - gateway_name=args.gateway_name, - trtype=trtype, - adrfam=adrfam, - traddr=traddr, - trsvcid=args.trsvcid, - ) ret = self.stub.delete_listener(req) - self.logger.info(f"Deleted {traddr}:{args.trsvcid} from {args.subnqn}: {ret.status}") - except ValueError as err: - self.logger.error(f"{err}") - self.logger.info(f"Deleted {traddr}:{args.trsvcid} from {args.subnqn}: {False}") - raise except Exception as ex: - self.logger.info(f"Deleted {traddr}:{args.trsvcid} from {args.subnqn}: {False}") - raise - - @cli.cmd() - def get_subsystems(self, args): - """Gets subsystems.""" - subsystems = json_format.MessageToJson( - self.stub.get_subsystems(pb2.get_subsystems_req()), - indent=4, including_default_value_fields=True, + ret = pb2.req_status(status = errno.EINVAL, + error_message = f"Failure deleting listener {traddr}:{args.trsvcid} from {args.subsystem}:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Deleting listener {traddr}:{args.trsvcid} from {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, preserving_proto_field_name=True) - # The address family enum values are lower case, convert them for display - subsystems = subsystems.replace('"adrfam": "ipv4"', '"adrfam": "IPv4"') - subsystems = subsystems.replace('"adrfam": "ipv6"', '"adrfam": "IPv6"') - self.logger.info(f"Get subsystems:\n{subsystems}") - - @cli.cmd() - def get_spdk_nvmf_log_flags_and_level(self, args): - """Gets spdk nvmf log levels and flags""" - req = pb2.get_spdk_nvmf_log_flags_and_level_req() - ret = self.stub.get_spdk_nvmf_log_flags_and_level(req) - formatted_flags_log_level = json.dumps(json.loads(ret.flags_level), indent=4) - self.logger.info( - f"Get SPDK nvmf log flags and level:\n{formatted_flags_log_level}") - - @cli.cmd() - def disable_spdk_nvmf_logs(self, args): - """Disables spdk nvmf logs and flags""" - req = pb2.disable_spdk_nvmf_logs_req() - ret = self.stub.disable_spdk_nvmf_logs(req) - self.logger.info( - f"Disable SPDK nvmf logs: {ret.status}") + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def listener_list(self, args): + """List listeners""" + + out_func, err_func = self.get_output_functions(args) + if args.gateway_name != None: + self.cli.parser.error("--gateway-name argument is not allowed for list command") + if args.traddr != None: + self.cli.parser.error("--traddr argument is not allowed for list command") + if args.trtype: + self.cli.parser.error("--trtype argument is not allowed for list command") + if args.adrfam: + self.cli.parser.error("--adrfam argument is not allowed for list command") + if args.trsvcid: + self.cli.parser.error("--trsvcid argument is not allowed for list command") + + listeners_info = None + try: + listeners_info = self.stub.list_listeners(pb2.list_listeners_req(subsystem=args.subsystem)) + except Exception as ex: + listeners_info = pb2.listeners_info(status = errno.EINVAL, error_message = f"Failure listing listeners:\n{ex}", listeners=[]) + + if args.format == "text": + if listeners_info.status == 0: + listeners_list = [] + for l in listeners_info.listeners: + adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, l.adrfam) + adrfam = self.format_adrfam(adrfam) + trtype = GatewayEnumUtils.get_key_from_value(pb2.TransportType, l.trtype) + listeners_list.append([l.gateway_name, trtype, adrfam, f"{l.traddr}:{l.trsvcid}"]) + if len(listeners_list) > 0: + listeners_out = tabulate(listeners_list, + headers = ["Gateway", "Transport", "Address Family", "Address"], + tablefmt="fancy_grid") + out_func(f"Listeners for {args.subsystem}:\n{listeners_out}") + else: + out_func(f"No listeners for {args.subsystem}") + else: + err_func(f"{listeners_info.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + listeners_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return listeners_info.status @cli.cmd([ - argument("-l", "--log_level", \ - help="SPDK nvmf log level (ERROR, WARNING, NOTICE, INFO, DEBUG)", required=False), - argument("-p", "--log_print_level", \ - help="SPDK nvmf log print level (ERROR, WARNING, NOTICE, INFO, DEBUG)", \ - required=False), + argument("listener_command", help="listener sub-command", choices=["add", "del", "delete", "remove", "list"]), + argument("--subsystem", "-n", help="Subsystem NQN", required=True), + argument("--gateway-name", "-g", help="Gateway name", required=False), + argument("--trtype", "-t", help="Transport type", default="", choices=get_enum_keys_list(pb2.TransportType)), + argument("--adrfam", "-f", help="Address family", default="", choices=get_enum_keys_list(pb2.AddressFamily)), + argument("--traddr", "-a", help="NVMe host IP", required=False), + argument("--trsvcid", "-s", help="Port number", type=int, required=False), ]) - def set_spdk_nvmf_logs(self, args): - """Set spdk nvmf log and print levels""" - log_level = None - print_level = None - if args.log_level: - log_level = args.log_level.upper() - if args.log_print_level: - print_level = args.log_print_level.upper() + def listener(self, args): + """Listeners commands""" + if args.listener_command == "add": + return self.listener_add(args) + elif args.listener_command == "del" or args.listener_command == "delete" or args.listener_command == "remove": + return self.listener_del(args) + elif args.listener_command == "list": + return self.listener_list(args) + assert False + + def host_add(self, args): + """Add a host to a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=args.host) try: - req = pb2.set_spdk_nvmf_logs_req(log_level=log_level, print_level=print_level) - ret = self.stub.set_spdk_nvmf_logs(req) - self.logger.info(f"Set SPDK nvmf logs: {ret.status}") - except ValueError as err: - self.logger.error(f"{err}") - self.logger.info(f"Set SPDK nvmf logs: {False}") - raise + ret = self.stub.add_host(req) except Exception as ex: - self.logger.info(f"Set SPDK nvmf logs: {False}") - raise + if args.host == "*": + errmsg = f"Failure allowing open host access to {args.subsystem}" + else: + errmsg = f"Failure adding host {args.host} to {args.subsystem}" + ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") - @cli.cmd() - def get_gateway_info(self, args): - """Get gateway's info""" - ver = os.getenv("NVMEOF_VERSION") - req = pb2.get_gateway_info_req(cli_version=ver) - gw_info = json_format.MessageToJson( - self.stub.get_gateway_info(req), + if args.format == "text": + if ret.status == 0: + if args.host == "*": + out_func(f"Allowing open host access to {args.subsystem}: Successful") + else: + out_func(f"Adding host {args.host} to {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def host_del(self, args): + """Delete a host from a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + req = pb2.remove_host_req(subsystem_nqn=args.subsystem, host_nqn=args.host) + + try: + ret = self.stub.remove_host(req) + except Exception as ex: + if args.host == "*": + errmsg = f"Failure disabling open host access to {args.subsystem}" + else: + errmsg = f"Failure removing host {args.host} access to {args.subsystem}" + ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + + if args.format == "text": + if ret.status == 0: + if args.host == "*": + out_func(f"Disabling open host access to {args.subsystem}: Successful") + else: + out_func(f"Removing host {args.host} access from {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def host_list(self, args): + """List a host for a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + if args.host != None: + self.cli.parser.error("--host argument is not allowed for list command") + + hosts_info = None + try: + hosts_info = self.stub.list_hosts(pb2.list_hosts_req(subsystem=args.subsystem)) + except Exception as ex: + hosts_info = pb2.hosts_info(status = errno.EINVAL, error_message = f"Failure listing hosts:\n{ex}", hosts=[]) + + if args.format == "text": + if hosts_info.status == 0: + hosts_list = [] + if hosts_info.allow_any_host: + hosts_list.append(["Any host"]) + for h in hosts_info.hosts: + hosts_list.append([h.nqn]) + if len(hosts_list) > 0: + hosts_out = tabulate(hosts_list, + headers = [f"Host NQN"], + tablefmt="fancy_grid", stralign="center") + out_func(f"Hosts allowed to access {args.subsystem}:\n{hosts_out}") + else: + out_func("No hosts are allowed to access {args.subsystem}") + else: + err_func(f"{hosts_info.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + hosts_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return hosts_info.status + + @cli.cmd([ + argument("host_command", help="host sub-command", choices=["add", "del", "delete", "remove", "list"]), + argument("--subsystem", "-n", help="Subsystem NQN", required=True), + argument("--host", "-t", help="Host NQN", required=False), + ]) + def host(self, args): + """Hosts commands""" + if args.host_command == "add": + return self.host_add(args) + elif args.host_command == "del" or args.host_command == "delete" or args.host_command == "remove": + return self.host_del(args) + elif args.host_command == "list": + return self.host_list(args) + assert False + + def connection_list(self, args): + """List connections for a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + connections_info = None + try: + connections_info = self.stub.list_connections(pb2.list_connections_req(subsystem=args.subsystem)) + except Exception as ex: + connections_info = pb2.connections_info(status = errno.EINVAL, + error_message = f"Failure listing hosts:\n{ex}", connections=[]) + + if args.format == "text": + if connections_info.status == 0: + connections_list = [] + for conn in connections_info.connections: + connections_list.append([conn.nqn, + f"{conn.traddr}:{conn.trsvcid}" if conn.connected else "", + "Yes" if conn.connected else "No", + conn.qpairs_count if conn.connected else "", + conn.controller_id if conn.connected else ""]) + if len(connections_list) > 0: + connections_out = tabulate(connections_list, + headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID"], + tablefmt="fancy_grid") + out_func(f"Connections for {args.subsystem}:\n{connections_out}") + else: + out_func(f"No connections for {args.subsystem}") + else: + err_func(f"{connections_info.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + connections_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return connections_info.status + + @cli.cmd([ + argument("connection_command", help="connection sub-command", choices=["list"]), + argument("--subsystem", "-n", help="Subsystem NQN", required=True), + ]) + def connection(self, args): + """Connections commands""" + if args.connection_command == "list": + return self.connection_list(args) + assert False + + def ns_add(self, args): + """Adds a namespace to a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + if args.block_size == None: + args.block_size = 512 + if args.load_balancing_group == None: + args.load_balancing_group = 1 + if args.load_balancing_group and args.load_balancing_group < 0: + self.cli.parser.error("load-balancing-group value must be positive") + if args.nsid and args.nsid <= 0: + self.cli.parser.error("nsid value must be positive") + if args.size != None: + self.cli.parser.error("--size argument is not allowed for add command") + if not args.rbd_pool: + self.cli.parser.error("--rbd-pool argument is mandatory for add command") + if not args.rbd_image: + self.cli.parser.error("--rbd-image argument is mandatory for add command") + if args.block_size <= 0: + self.cli.parser.error("block-size value must be positive") + if args.load_balancing_group <= 0: + self.cli.parser.error("load-balancing-group value must be positive") + if args.rw_ios_per_second != None: + self.cli.parser.error("--rw-ios-per-second argument is not allowed for add command") + if args.rw_megabytes_per_second != None: + self.cli.parser.error("--rw-megabytes-per-second argument is not allowed for add command") + if args.r_megabytes_per_second != None: + self.cli.parser.error("--r-megabytes-per-second argument is not allowed for add command") + if args.w_megabytes_per_second != None: + self.cli.parser.error("--w-megabytes-per-second argument is not allowed for add command") + + req = pb2.namespace_add_req(rbd_pool_name=args.rbd_pool, + rbd_image_name=args.rbd_image, + subsystem_nqn=args.subsystem, + nsid=args.nsid, + block_size=args.block_size, + uuid=args.uuid, + anagrpid=args.load_balancing_group) + try: + ret = self.stub.namespace_add(req) + except Exception as ex: + nsid_msg = "" + if args.nsid: + nsid_msg = f"using NSID {args.nsid} " + errmsg = f"Failure adding namespace {nsid_msg}to {args.subsystem}" + ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Adding namespace {ret.nsid} to {args.subsystem}, load balancing group {args.load_balancing_group}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def ns_del(self, args): + """Deletes a namespace from a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + if not args.nsid: + self.cli.parser.error("--nsid argument is mandatory for del command") + if args.nsid < 0: + self.cli.parser.error("nsid value must be positive") + if args.size != None: + self.cli.parser.error("--size argument is not allowed for del command") + if args.block_size != None: + self.cli.parser.error("--block-size argument is not allowed for del command") + if args.rbd_pool != None: + self.cli.parser.error("--rbd-pool argument is not allowed for del command") + if args.rbd_image != None: + self.cli.parser.error("--rbd-image argument is not allowed for del command") + if args.uuid != None: + self.cli.parser.error("--uuid argument is not allowed for del command") + if args.load_balancing_group: + self.cli.parser.error("--load-balancing-group argument is not allowed for del command") + if args.rw_ios_per_second != None: + self.cli.parser.error("--rw-ios-per-second argument is not allowed for del command") + if args.rw_megabytes_per_second != None: + self.cli.parser.error("--rw-megabytes-per-second argument is not allowed for del command") + if args.r_megabytes_per_second != None: + self.cli.parser.error("--r-megabytes-per-second argument is not allowed for del command") + if args.w_megabytes_per_second != None: + self.cli.parser.error("--w-megabytes-per-second argument is not allowed for del command") + + try: + ret = self.stub.namespace_delete(pb2.namespace_delete_req(subsystem_nqn=args.subsystem, nsid=args.nsid)) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure deleting namespace:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Deleting namespace {args.nsid} from {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def ns_resize(self, args): + """Resizes a namespace.""" + + out_func, err_func = self.get_output_functions(args) + if not args.nsid: + self.cli.parser.error("--nsid argument is mandatory for resize command") + if args.nsid < 0: + self.cli.parser.error("nsid value must be positive") + if not args.size: + self.cli.parser.error("--size argument is mandatory for resize command") + if args.size < 0: + self.cli.parser.error("size value must be positive") + if args.block_size != None: + self.cli.parser.error("--block-size argument is not allowed for resize command") + if args.rbd_pool != None: + self.cli.parser.error("--rbd-pool argument is not allowed for resize command") + if args.rbd_image != None: + self.cli.parser.error("--rbd-image argument is not allowed for resize command") + if args.uuid != None: + self.cli.parser.error("--uuid argument is not allowed for resize command") + if args.load_balancing_group != None: + self.cli.parser.error("--load-balancing-group argument is not allowed for resize command") + if args.rw_ios_per_second != None: + self.cli.parser.error("--rw-ios-per-second argument is not allowed for resize command") + if args.rw_megabytes_per_second != None: + self.cli.parser.error("--rw-megabytes-per-second argument is not allowed for resize command") + if args.r_megabytes_per_second != None: + self.cli.parser.error("--r-megabytes-per-second argument is not allowed for resize command") + if args.w_megabytes_per_second != None: + self.cli.parser.error("--w-megabytes-per-second argument is not allowed for resize command") + + try: + ret = self.stub.namespace_resize(pb2.namespace_resize_req(subsystem_nqn=args.subsystem, nsid=args.nsid, new_size=args.size)) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure resizing namespace:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Resizing namespace {args.nsid} on {args.subsystem} to {args.size} MiB: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def format_size(self, sz): + if sz < 1024: + return str(sz) + " B" + if sz < 1024 * 1024: + sz = sz / 1024.0 + if sz == int(sz): + sz = int(sz) + return f"{sz} KiB" + return f"{sz:2.1f} KiB" + if sz < 1024 * 1024 * 1024: + sz = sz / (1024.0 * 1024.0) + if sz == int(sz): + sz = int(sz) + return f"{sz} MiB" + return f"{sz:2.1f} MiB" + if sz < 1024 * 1024 * 1024 * 1024: + sz = sz / (1024.0 * 1024.0 * 1024.0) + if sz == int(sz): + sz = int(sz) + return f"{sz} GiB" + return f"{sz:2.1f} GiB" + sz = sz / (1024.0 * 1024.0 * 1024.0 * 1024.0) + if sz == int(sz): + sz = int(sz) + return f"{sz} TiB" + return f"{sz:2.1f} TiB" + + def ns_list(self, args): + """Lists namespaces on a subsystem.""" + + out_func, err_func = self.get_output_functions(args) + if args.nsid and args.nsid < 0: + self.cli.parser.error("nsid value must be positive") + if args.size != None: + self.cli.parser.error("--size argument is not allowed for list command") + if args.block_size != None: + self.cli.parser.error("--block-size argument is not allowed for list command") + if args.rbd_pool != None: + self.cli.parser.error("--rbd-pool argument is not allowed for list command") + if args.rbd_image != None: + self.cli.parser.error("--rbd-image argument is not allowed for list command") + if args.load_balancing_group != None: + self.cli.parser.error("--load-balancing-group argument is not allowed for list command") + if args.rw_ios_per_second != None: + self.cli.parser.error("--rw-ios-per-second argument is not allowed for list command") + if args.rw_megabytes_per_second != None: + self.cli.parser.error("--rw-megabytes-per-second argument is not allowed for list command") + if args.r_megabytes_per_second != None: + self.cli.parser.error("--r-megabytes-per-second argument is not allowed for list command") + if args.w_megabytes_per_second != None: + self.cli.parser.error("--w-megabytes-per-second argument is not allowed for list command") + + try: + namespaces_info = self.stub.list_namespaces(pb2.list_namespaces_req(subsystem=args.subsystem, + nsid=args.nsid, uuid=args.uuid)) + except Exception as ex: + namespaces_info = pb2.namespaces_info(status = errno.EINVAL, error_message = f"Failure listing namespaces:\n{ex}") + + if args.format == "text": + if namespaces_info.status == 0: + if args.nsid and len(namespaces_info.namespaces) > 1: + err_func(f"Got more than one namespace for NSID {args.nsid}") + if args.uuid and len(namespaces_info.namespaces) > 1: + err_func(f"Got more than one namespace for UUID {args.uuid}") + namespaces_list = [] + for ns in namespaces_info.namespaces: + if args.nsid and args.nsid != ns.nsid: + err_func("Failure listing namespace {args.nsid}: Got namespace {ns.nsid} instead") + return errno.ENODEV + if args.uuid and args.uuid != ns.uuid: + err_func("Failure listing namespace with UUID {args.uuid}: Got namespace {ns.uuid} instead") + return errno.ENODEV + if ns.load_balancing_group == 0: + lb_group = "" + else: + lb_group = str(ns.load_balancing_group) + namespaces_list.append([ns.nsid, + ns.bdev_name, + ns.rbd_image_name, + ns.rbd_pool_name, + self.format_size(ns.rbd_image_size), + self.format_size(ns.block_size), + ns.uuid, + lb_group, + self.get_qos_limit_str_value(ns.rw_ios_per_second), + self.get_qos_limit_str_value(ns.rw_mbytes_per_second), + self.get_qos_limit_str_value(ns.r_mbytes_per_second), + self.get_qos_limit_str_value(ns.w_mbytes_per_second)]) + + if len(namespaces_list) > 0: + namespaces_out = tabulate(namespaces_list, + headers = ["NSID", "Bdev Name", "RBD Image", "RBD Pool", + "Image Size", "Block Size", "UUID", "Load Balancing\nGroup", + "R/W IOs\nper\nsecond", "R/W MBs\nper\nsecond", + "Read MBs\nper\nsecond", "Write MBs\nper\nsecond"], + tablefmt="fancy_grid") + if args.nsid: + prefix = f"Namespace {args.nsid} in" + elif args.uuid: + prefix = f"Namespace with UUID {args.uuid} in" + else: + prefix = "Namespaces in" + out_func(f"{prefix} subsystem {args.subsystem}:\n{namespaces_out}") + else: + if args.nsid: + out_func(f"No namespace {args.nsid} in subsystem {args.subsystem}") + elif args.uuid: + out_func(f"No namespace with UUID {args.uuid} in subsystem {args.subsystem}") + else: + out_func(f"No namespaces in subsystem {args.subsystem}") + else: + err_func(f"{namespaces_info.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + namespaces_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return namespaces_info.status + + def ns_get_io_stats(self, args): + """Get namespace IO statistics.""" + + out_func, err_func = self.get_output_functions(args) + if not args.nsid: + self.cli.parser.error("--nsid argument is mandatory for get_io_stats command") + if args.nsid < 0: + self.cli.parser.error("nsid value must be positive") + if args.size != None: + self.cli.parser.error("--size argument is not allowed for get_io_stats command") + if args.block_size != None: + self.cli.parser.error("--block-size argument is not allowed for get_io_stats command") + if args.rbd_pool != None: + self.cli.parser.error("--rbd-pool argument is not allowed for get_io_stats command") + if args.rbd_image != None: + self.cli.parser.error("--rbd-image argument is not allowed for get_io_stats command") + if args.uuid != None: + self.cli.parser.error("--uuid argument is not allowed for get_io_stats command") + if args.load_balancing_group != None: + self.cli.parser.error("--load-balancing-group argument is not allowed for get_io_stats command") + if args.rw_ios_per_second != None: + self.cli.parser.error("--rw-ios-per-second argument is not allowed for get_io_stats command") + if args.rw_megabytes_per_second != None: + self.cli.parser.error("--rw-megabytes-per-second argument is not allowed for get_io_stats command") + if args.r_megabytes_per_second != None: + self.cli.parser.error("--r-megabytes-per-second argument is not allowed for get_io_stats command") + if args.w_megabytes_per_second != None: + self.cli.parser.error("--w-megabytes-per-second argument is not allowed for get_io_stats command") + + try: + get_stats_req = pb2.namespace_get_io_stats_req(subsystem_nqn=args.subsystem, nsid=args.nsid) + ns_io_stats = self.stub.namespace_get_io_stats(get_stats_req) + except Exception as ex: + ns_io_stats = pb2.namespaces_info(status = errno.EINVAL, error_message = f"Failure getting namespace's IO stats:\n{ex}") + + if ns_io_stats.status == 0 and ns_io_stats.subsystem_nqn != args.subsystem: + ns_io_stats.status = errno.EINVAL + ns_io_stats.error_message = f"Failure getting namespace's IO stats: Returned subsystem {ns_io_stats.subsystem_nqn} differs from requested one {args.subsystem}" + + if args.format == "text": + if ns_io_stats.status == 0: + stats_list = [] + stats_list.append(["Tick Rate", ns_io_stats.tick_rate]) + stats_list.append(["Ticks", ns_io_stats.ticks]) + stats_list.append(["Bytes Read", ns_io_stats.bytes_read]) + stats_list.append(["Num Read Ops", ns_io_stats.num_read_ops]) + stats_list.append(["Bytes Written", ns_io_stats.bytes_written]) + stats_list.append(["Num Write Ops", ns_io_stats.num_write_ops]) + stats_list.append(["Bytes Unmapped", ns_io_stats.bytes_unmapped]) + stats_list.append(["Num Unmap Ops", ns_io_stats.num_unmap_ops]) + stats_list.append(["Read Latency Ticks", ns_io_stats.read_latency_ticks]) + stats_list.append(["Max Read Latency Ticks", ns_io_stats.max_read_latency_ticks]) + stats_list.append(["Min Read Latency Ticks", ns_io_stats.min_read_latency_ticks]) + stats_list.append(["Write Latency Ticks", ns_io_stats.write_latency_ticks]) + stats_list.append(["Max Write Latency Ticks", ns_io_stats.max_write_latency_ticks]) + stats_list.append(["Min Write Latency Ticks", ns_io_stats.min_write_latency_ticks]) + stats_list.append(["Unmap Latency Ticks", ns_io_stats.unmap_latency_ticks]) + stats_list.append(["Max Unmap Latency Ticks", ns_io_stats.max_unmap_latency_ticks]) + stats_list.append(["Min Unmap Latency Ticks", ns_io_stats.min_unmap_latency_ticks]) + stats_list.append(["Copy Latency Ticks", ns_io_stats.copy_latency_ticks]) + stats_list.append(["Max Copy Latency Ticks", ns_io_stats.max_copy_latency_ticks]) + stats_list.append(["Min Copy Latency Ticks", ns_io_stats.min_copy_latency_ticks]) + stats_list.append(["IO Error", str(ns_io_stats.io_error)]) + + stats_out = tabulate(stats_list, headers = ["Stat", "Value"], tablefmt="fancy_grid") + out_func(f"IO statistics for namespace {args.nsid} on {args.subsystem}, bdev {ns_io_stats.bdev_name}:\n{stats_out}") + else: + err_func(f"{ns_io_stats.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ns_io_stats, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ns_io_stats.status + + def ns_change_load_balancing_group(self, args): + """Change namespace load balancing group.""" + + out_func, err_func = self.get_output_functions(args) + if not args.nsid: + self.cli.parser.error("--nsid argument is mandatory for change_load_balancing_group command") + if args.nsid < 0: + self.cli.parser.error("nsid value must be positive") + if args.load_balancing_group == None: + self.cli.parser.error("--load-balancing-group argument is mandatory for change_load_balancing_group command") + if args.load_balancing_group <= 0: + self.cli.parser.error("load-balancing-group value must be positive") + if args.size != None: + self.cli.parser.error("--size argument is not allowed for change_load_balancing_group command") + if args.block_size != None: + self.cli.parser.error("--block-size argument is not allowed for change_load_balancing_group command") + if args.rbd_pool != None: + self.cli.parser.error("--rbd-pool argument is not allowed for change_load_balancing_group command") + if args.rbd_image != None: + self.cli.parser.error("--rbd-image argument is not allowed for change_load_balancing_group command") + if args.uuid != None: + self.cli.parser.error("--uuid argument is not allowed for change_load_balancing_group command") + if args.rw_ios_per_second != None: + self.cli.parser.error("--rw-ios-per-second argument is not allowed for change_load_balancing_group command") + if args.rw_megabytes_per_second != None: + self.cli.parser.error("--rw-megabytes-per-second argument is not allowed for change_load_balancing_group command") + if args.r_megabytes_per_second != None: + self.cli.parser.error("--r-megabytes-per-second argument is not allowed for change_load_balancing_group command") + if args.w_megabytes_per_second != None: + self.cli.parser.error("--w-megabytes-per-second argument is not allowed for change_load_balancing_group command") + + try: + change_lb_group_req = pb2.namespace_change_load_balancing_group_req(subsystem_nqn=args.subsystem, + nsid=args.nsid, + anagrpid=args.load_balancing_group) + ret = self.stub.namespace_change_load_balancing_group(change_lb_group_req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure changing namespace load balancing group:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Changing load balancing group of namespace {args.nsid} in {args.subsystem} to {args.load_balancing_group}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, indent=4, including_default_value_fields=True, preserving_proto_field_name=True) - self.logger.info(f"Gateway info:\n{gw_info}") + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False + + return ret.status + + def get_qos_limit_str_value(self, qos_limit): + if qos_limit == 0: + return "unlimited" + else: + return str(qos_limit) + + def ns_set_qos(self, args): + """Set namespace QOS limits.""" + + out_func, err_func = self.get_output_functions(args) + if not args.nsid: + self.cli.parser.error("--nsid argument is mandatory for set_qos command") + if args.nsid < 0: + self.cli.parser.error("nsid value must be positive") + if args.load_balancing_group != None: + self.cli.parser.error("--load-balancing-group argument is not allowed for set_qos command") + if args.size != None: + self.cli.parser.error("--size argument is not allowed for set_qos command") + if args.block_size != None: + self.cli.parser.error("--block-size argument is not allowed for set_qos command") + if args.rbd_pool != None: + self.cli.parser.error("--rbd-pool argument is not allowed for set_qos command") + if args.rbd_image != None: + self.cli.parser.error("--rbd-image argument is not allowed for set_qos command") + if args.uuid != None: + self.cli.parser.error("--uuid argument is not allowed for set_qos command") + rw_ios_per_second_is_set = False if args.rw_ios_per_second == None else True + if args.format == "text" and args.rw_ios_per_second and (args.rw_ios_per_second % 1000) != 0: + rounded_rate = int((args.rw_ios_per_second + 1000) / 1000) * 1000 + err_func(f"IOs per second {args.rw_ios_per_second} will be rounded up to {rounded_rate}") + rw_megabytes_per_second_is_set = False if args.rw_megabytes_per_second == None else True + r_megabytes_per_second_is_set = False if args.r_megabytes_per_second == None else True + w_megabytes_per_second_is_set = False if args.w_megabytes_per_second == None else True + + try: + set_qos_req = pb2.namespace_set_qos_req(subsystem_nqn=args.subsystem, nsid=args.nsid, + rw_ios_per_second_is_set=rw_ios_per_second_is_set, + rw_ios_per_second=args.rw_ios_per_second, + rw_mbytes_per_second_is_set=rw_megabytes_per_second_is_set, + rw_mbytes_per_second=args.rw_megabytes_per_second, + r_mbytes_per_second_is_set=r_megabytes_per_second_is_set, + r_mbytes_per_second=args.r_megabytes_per_second, + w_mbytes_per_second_is_set=w_megabytes_per_second_is_set, + w_mbytes_per_second=args.w_megabytes_per_second) + ret = self.stub.namespace_set_qos_limits(set_qos_req) + except Exception as ex: + ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure setting namespaces QOS limits:\n{ex}") + + if args.format == "text": + if ret.status == 0: + out_func(f"Setting QOS limits of namespace {args.nsid} in {args.subsystem}: Successful") + else: + err_func(f"{ret.error_message}") + elif args.format == "json" or args.format == "yaml": + ret_str = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True) + if args.format == "json": + out_func(f"{ret_str}") + else: + obj = json.loads(ret_str) + out_func(yaml.dump(obj)) + else: + assert False -def main(args=None): + return ret.status + + ns_args_list = [ + argument("ns_command", help="namespace sub-command", + choices=["add", "del", "delete", "remove", "resize", "list", "get_io_stats", + "change_load_balancing_group", "set_qos"]), + argument("--subsystem", "-n", help="Subsystem NQN", required=True), + argument("--rbd-pool", "-p", help="RBD pool name"), + argument("--rbd-image", "-i", help="RBD image name"), + argument("--block-size", "-s", help="Block size", type=int), + argument("--uuid", "-u", help="UUID"), + argument("--nsid", help="Namespace ID", type=int), + argument("--load-balancing-group", "-l", help="Load balancing group", type=int), + argument("--size", help="New size in MiB", type=int), + argument("--rw-ios-per-second", help="R/W IOs per second limit, 0 means unlimited", type=int), + argument("--rw-megabytes-per-second", help="R/W megabytes per second limit, 0 means unlimited", type=int), + argument("--r-megabytes-per-second", help="Read megabytes per second limit, 0 means unlimited", type=int), + argument("--w-megabytes-per-second", help="Write megabytes per second limit, 0 means unlimited", type=int), + ] + @cli.cmd(ns_args_list) + def ns(self, args): + """Namespace commands""" + if args.ns_command == "add": + return self.ns_add(args) + elif args.ns_command == "del" or args.ns_command == "delete" or args.ns_command == "remove": + return self.ns_del(args) + elif args.ns_command == "resize": + return self.ns_resize(args) + elif args.ns_command == "list": + return self.ns_list(args) + elif args.ns_command == "get_io_stats": + return self.ns_get_io_stats(args) + elif args.ns_command == "change_load_balancing_group": + return self.ns_change_load_balancing_group(args) + elif args.ns_command == "set_qos": + return self.ns_set_qos(args) + assert False + + @cli.cmd(ns_args_list) + def namespace(self, args): + """Namespace commands""" + return self.ns(args) + +def main(args=None) -> int: client = GatewayClient() parsed_args = client.cli.parser.parse_args(args) if parsed_args.subcommand is None: client.cli.parser.print_help() - return 0 + return -1 + server_address = parsed_args.server_address server_port = parsed_args.server_port client_key = parsed_args.client_key @@ -446,8 +1638,8 @@ def main(args=None): server_cert = parsed_args.server_cert client.connect(server_address, server_port, client_key, client_cert, server_cert) call_function = getattr(client, parsed_args.func.__name__) - call_function(parsed_args) - + rc = call_function(parsed_args) + return rc if __name__ == "__main__": sys.exit(main()) diff --git a/control/config.py b/control/config.py index 3ef2ab444..a0150a552 100644 --- a/control/config.py +++ b/control/config.py @@ -9,6 +9,34 @@ import configparser +class GatewayEnumUtils: + def get_value_from_key(e_type, keyval, ignore_case = False): + val = None + try: + key_index = e_type.keys().index(keyval) + val = e_type.values()[key_index] + except ValueError: + pass + except IndexError: + pass + + if ignore_case and val == None and type(keyval) == str: + val = get_value_from_key(e_type, keyval.lower(), False) + if ignore_case and val == None and type(keyval) == str: + val = get_value_from_key(e_type, keyval.upper(), False) + + return val + + def get_key_from_value(e_type, val): + keyval = None + try: + val_index = e_type.values().index(val) + keyval = e_type.keys()[val_index] + except ValueError: + pass + except IndexError: + pass + return keyval class GatewayConfig: """Loads and returns config file settings. diff --git a/control/discovery.py b/control/discovery.py index c1ad81b1b..147893074 100644 --- a/control/discovery.py +++ b/control/discovery.py @@ -13,7 +13,7 @@ import logging from .config import GatewayConfig from .state import GatewayState, LocalGatewayState, OmapGatewayState, GatewayStateHandler -from .grpc import GatewayEnumUtils +from .config import GatewayEnumUtils from .proto import gateway_pb2 as pb2 import rados @@ -738,10 +738,11 @@ def reply_get_log_page(self, conn, data, cmd_id): log_entry.controller_id = 0xffff log_entry.asqsz = 128 # transport service indentifier + str_trsvcid = str(allow_listeners[log_entry_counter]["trsvcid"]) log_entry.trsvcid = (c_ubyte * 32)(*[c_ubyte(x) for x \ - in allow_listeners[log_entry_counter]["trsvcid"].encode()]) - log_entry.trsvcid[len(allow_listeners[log_entry_counter]["trsvcid"]):] = \ - [c_ubyte(0x20)] * (32 - len(allow_listeners[log_entry_counter]["trsvcid"])) + in str_trsvcid.encode()]) + log_entry.trsvcid[len(str_trsvcid):] = \ + [c_ubyte(0x20)] * (32 - len(str_trsvcid)) # NVM subsystem qualified name log_entry.subnqn = (c_ubyte * 256)(*[c_ubyte(x) for x \ in allow_listeners[log_entry_counter]["nqn"].encode()]) diff --git a/control/grpc.py b/control/grpc.py index 776ff1f4f..0180f63dd 100644 --- a/control/grpc.py +++ b/control/grpc.py @@ -20,44 +20,17 @@ import spdk.rpc.bdev as rpc_bdev import spdk.rpc.nvmf as rpc_nvmf import spdk.rpc.log as rpc_log +from spdk.rpc.client import JSONRPCException from google.protobuf import json_format from .proto import gateway_pb2 as pb2 from .proto import gateway_pb2_grpc as pb2_grpc from .config import GatewayConfig +from .config import GatewayEnumUtils from .state import GatewayState MAX_ANA_GROUPS = 4 -class GatewayEnumUtils: - def get_value_from_key(e_type, keyval, ignore_case = False): - val = None - try: - key_index = e_type.keys().index(keyval) - val = e_type.values()[key_index] - except ValueError: - pass - except IndexError: - pass - - if ignore_case and val == None and type(keyval) == str: - val = get_value_from_key(e_type, keyval.lower(), False) - if ignore_case and val == None and type(keyval) == str: - val = get_value_from_key(e_type, keyval.upper(), False) - - return val - - def get_key_from_value(e_type, val): - keyval = None - try: - val_index = e_type.values().index(val) - keyval = e_type.keys()[val_index] - except ValueError: - pass - except IndexError: - pass - return keyval - class GatewayService(pb2_grpc.GatewayServicer): """Implements gateway service interface. @@ -110,6 +83,28 @@ def __init__(self, config, gateway_state, omap_lock, spdk_rpc_client) -> None: self.gateway_group = self.config.get("gateway", "group") self._init_cluster_context() + def parse_json_exeption(self, ex): + if type(ex) != JSONRPCException: + return None + + json_error_text = "Got JSON-RPC error response" + rsp = None + try: + resp_index = ex.message.find(json_error_text) + if resp_index >= 0: + resp_str = ex.message[resp_index + len(json_error_text) :] + resp_index = resp_str.find("response:") + if resp_index >= 0: + resp_str = resp_str[resp_index + len("response:") :] + resp = json.loads(resp_str) + except Exception as jsex: + self.logger.error(f"Got exception parsing JSon exception: {jsex}") + pass + if resp: + if resp["code"] < 0: + resp["code"] = -resp["code"] + return resp + def _init_cluster_context(self) -> None: """Init cluster context management variables""" self.clusters = {} @@ -184,11 +179,23 @@ def create_bdev_safe(self, request, context=None): ) self.logger.info(f"create_bdev: {bdev_name}") except Exception as ex: - self.logger.error(f"create_bdev failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.bdev() + errmsg = f"create_bdev {name} failed with:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure creating bdev {name}: {resp['message']}" + return pb2.bdev_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not bdev_name: + errmsg = f"Can't create bdev {name}" + self.logger.error(errmsg) + return pb2.bdev_status(status=errno.EINVAL, error_message=errmsg) + + if name != bdev_name: + self.logger.warning(f"Created bdev name {bdev_name} differs from requested name {name}") if context: # Update gateway state @@ -197,35 +204,46 @@ def create_bdev_safe(self, request, context=None): request, preserving_proto_field_name=True, including_default_value_fields=True) self.gateway_state.add_bdev(bdev_name, json_req) except Exception as ex: - self.logger.error( - f"Error persisting create_bdev {bdev_name}: {ex}") - raise + errmsg = f"Error persisting bdev {name}" + self.logger.error(f"{errmsg}:\n{ex}") + return pb2.bdev_status(status=errno.EINVAL, error_message=errmsg) - return pb2.bdev(bdev_name=bdev_name, status=True) + return pb2.bdev_status(bdev_name=name, status=0, error_message=os.strerror(0)) def create_bdev(self, request, context=None): return self.execute_grpc_function(self.create_bdev_safe, request, context) - def resize_bdev_safe(self, request): + def resize_bdev_safe(self, bdev_name, new_size): """Resizes a bdev.""" - self.logger.info(f"Received request to resize bdev {request.bdev_name} to size {request.new_size} MiB") + self.logger.info(f"Received request to resize bdev {bdev_name} to size {new_size} MiB") try: ret = rpc_bdev.bdev_rbd_resize( self.spdk_rpc_client, - name=request.bdev_name, - new_size=request.new_size, + name=bdev_name, + new_size=new_size, ) - self.logger.info(f"resize_bdev: {request.bdev_name}: {ret}") + self.logger.info(f"resize_bdev {bdev_name}: {ret}") except Exception as ex: - self.logger.error(f"resize_bdev failed with: \n {ex}") - return pb2.req_status() - - return pb2.req_status(status=ret) - - def resize_bdev(self, request, context=None): + errmsg = f"Failure resizing bdev {bdev_name}:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure resizing bdev {bdev_name}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + if not ret: + errmsg = f"Failure resizing bdev {bdev_name}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + + return pb2.req_status(status=0, error_message=os.strerror(0)) + + def resize_bdev(self, bdev_name, new_size): with self.rpc_lock: - return self.resize_bdev_safe(request) + return self.resize_bdev_safe(bdev_name, new_size) def get_bdev_namespaces(self, bdev_name) -> list: ns_list = [] @@ -238,19 +256,25 @@ def get_bdev_namespaces(self, bdev_name) -> list: if ns["bdev_name"] == bdev_name: nsid = ns["nsid"] nqn = ns["subsystem_nqn"] - ns_list.insert(0, {"nqn" : nqn, "nsid" : nsid}) + ns_list.append({"nqn" : nqn, "nsid" : nsid}) except Exception as ex: self.logger.error(f"Got exception trying to get bdev {bdev_name} namespaces: {ex}") pass return ns_list - def delete_bdev_handle_exception(self, context, ex): - self.logger.error(f"delete_bdev failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + def remove_bdev_from_state(self, bdev_name, context): + if not context: + return pb2.req_status(status=0, error_message=os.strerror(0)) + + # Update gateway state + try: + self.gateway_state.remove_bdev(bdev_name) + except Exception as ex: + errmsg = f"Error persisting deletion of bdev {request.bdev_name}:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + return pb2.req_status(status=0, error_message=os.strerror(0)) def delete_bdev_safe(self, request, context=None): """Deletes a bdev.""" @@ -271,20 +295,17 @@ def delete_bdev_safe(self, request, context=None): continue if request.force: - self.logger.info(f"Will remove namespace {ns_nsid} from {ns_nqn} as it is using bdev {request.bdev_name}") + self.logger.warning(f"Will remove namespace {ns_nsid} from {ns_nqn} as it is using bdev {request.bdev_name}") try: - self.gateway_state.remove_namespace(ns_nqn, str(ns_nsid)) + self.remove_namespace_from_state(ns_nqn, ns_nsid, context) self.logger.info(f"Removed namespace {ns_nsid} from {ns_nqn}") except Exception as ex: self.logger.error(f"Error removing namespace {ns_nsid} from {ns_nqn}, will delete bdev {request.bdev_name} anyway: {ex}") pass else: - self.logger.error(f"Namespace {ns_nsid} from {ns_nqn} is still using bdev {request.bdev_name}. You need to either remove it or use the '--force' command line option") - req = {"name": request.bdev_name, "method": "bdev_rbd_delete", "req_id": 0} - ret = {"code": -errno.EBUSY, "message": os.strerror(errno.EBUSY)} - msg = "\n".join(["request:", "%s" % json.dumps(req, indent = 2), - "Got JSON-RPC error response", "response:", json.dumps(ret, indent = 2)]) - return self.delete_bdev_handle_exception(context, Exception(msg)) + errmsg = f"Failure deleting bdev {request.bdev_name}: Namespace {ns_nsid} from {ns_nqn} is still using it" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EBUSY, error_message=errmsg) try: ret = rpc_bdev.bdev_rbd_delete( @@ -293,18 +314,24 @@ def delete_bdev_safe(self, request, context=None): ) self.logger.info(f"delete_bdev {request.bdev_name}: {ret}") except Exception as ex: - return self.delete_bdev_handle_exception(context, ex) - - if context: - # Update gateway state - try: - self.gateway_state.remove_bdev(request.bdev_name) - except Exception as ex: - self.logger.error( - f"Error persisting delete_bdev {request.bdev_name}: {ex}") - raise - - return pb2.req_status(status=ret) + errmsg = f"Failure deleting bdev {request.bdev_name}:\n{ex}" + self.logger.error(errmsg) + self.remove_bdev_from_state(request.bdev_name, context) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure deleting bdev {request.bdev_name}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + errmsg = f"Failure deleting bdev {request.bdev_name}" + self.logger.error(errmsg) + self.remove_bdev_from_state(request.bdev_name, context) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + + return self.remove_bdev_from_state(request.bdev_name, context) def delete_bdev(self, request, context=None): return self.execute_grpc_function(self.delete_bdev_safe, request, context) @@ -312,6 +339,23 @@ def delete_bdev(self, request, context=None): def is_discovery_nqn(self, nqn) -> bool: return nqn == GatewayConfig.DISCOVERY_NQN + def subsystem_already_exists(self, context, nqn) -> bool: + if not context: + return False + state = self.gateway_state.local.get_state() + for key, val in state.items(): + if not key.startswith(self.gateway_state.local.SUBSYSTEM_PREFIX): + continue + try: + subsys = json.loads(val) + subnqn = subsys["subsystem_nqn"] + if subnqn == nqn: + return True + except Exception as ex: + self.logger.warning(f"Got exception while parsing {val}:\n{ex}") + continue + return False + def serial_number_already_used(self, context, serial) -> str: if not context: return None @@ -321,50 +365,63 @@ def serial_number_already_used(self, context, serial) -> str: continue try: subsys = json.loads(val) - sn = subsys["serial_number"] - if serial == sn: + if serial == subsys["serial_number"]: return subsys["subsystem_nqn"] - except Exception: - self.logger.warning("Got exception while parsing {val}: {ex}") + except Exception as ex: + self.logger.warning("Got exception while parsing {val}:\n{ex}") continue return None def create_subsystem_safe(self, request, context=None): """Creates a subsystem.""" + create_subsystem_error_prefix = f"Failure creating subsystem {request.subsystem_nqn}" + self.logger.info( f"Received request to create subsystem {request.subsystem_nqn}, enable_ha: {request.enable_ha}, ana reporting: {request.ana_reporting}, context: {context}") + errmsg = "" if self.is_discovery_nqn(request.subsystem_nqn): - raise Exception(f"Can't create a discovery subsystem") - if request.enable_ha == True and request.ana_reporting == False: - raise Exception(f"Validation Error: HA enabled but ANA-reporting is disabled ") + errmsg = f"{create_subsystem_error_prefix}: Can't create a discovery subsystem" + ret = pb2.req_status(status = errno.EINVAL, error_message = errmsg) + self.logger.error(f"{errmsg}") + return ret + if request.enable_ha and not request.ana_reporting: + errmsg = f"{create_subsystem_error_prefix}: HA is enabled but ANA reporting is disabled" + ret = pb2.req_status(status = errno.EINVAL, error_message = errmsg) + self.logger.error(f"{errmsg}") + return ret min_cntlid = self.config.getint_with_default("gateway", "min_controller_id", 1) max_cntlid = self.config.getint_with_default("gateway", "max_controller_id", 65519) + if min_cntlid > max_cntlid: + errmsg = f"{create_subsystem_error_prefix}: Min controller id {min_cntlid} is bigger than max controller id {max_cntlid}" + ret = pb2.req_status(status = errno.EINVAL, error_message = errmsg) + self.logger.error(f"{errmsg}") + return ret + if not request.serial_number: random.seed() randser = random.randint(2, 99999999999999) request.serial_number = f"SPDK{randser}" - self.logger.info(f"No serial number specified, will use {request.serial_number}") + self.logger.info(f"No serial number specified for {request.subsystem_nqn}, will use {request.serial_number}") + ret = False with self.omap_lock(context=context): + errmsg = "" try: - subsys_using_serial = self.serial_number_already_used(context, request.serial_number) - if subsys_using_serial: - self.logger.error(f"Serial number {request.serial_number} already used by subsystem {subsys_using_serial}") - req = {"subsystem_nqn": request.subsystem_nqn, - "serial_number": request.serial_number, - "max_namespaces": request.max_namespaces, - "ana_reporting": request.ana_reporting, - "enable_ha": request.enable_ha, - "method": "nvmf_create_subsystem", "req_id": 0} - ret = {"code": -errno.EEXIST, "message": f"Serial number {request.serial_number} already used by subsystem {subsys_using_serial}"} - msg = "\n".join(["request:", "%s" % json.dumps(req, indent=2), - "Got JSON-RPC error response", - "response:", - json.dumps(ret, indent=2)]) - raise Exception(msg) + subsys_using_serial = None + subsys_already_exists = self.subsystem_already_exists(context, request.subsystem_nqn) + if subsys_already_exists: + errmsg = f"Subsystem already exists" + else: + subsys_using_serial = self.serial_number_already_used(context, request.serial_number) + if subsys_using_serial: + errmsg = f"Serial number {request.serial_number} already used by subsystem {subsys_using_serial}" + if subsys_already_exists or subsys_using_serial: + errmsg = f"{create_subsystem_error_prefix}: {errmsg}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EEXIST, error_message=errmsg) ret = rpc_nvmf.nvmf_create_subsystem( self.spdk_rpc_client, nqn=request.subsystem_nqn, @@ -376,38 +433,87 @@ def create_subsystem_safe(self, request, context=None): ) self.logger.info(f"create_subsystem {request.subsystem_nqn}: {ret}") except Exception as ex: - self.logger.error(f"create_subsystem failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + errmsg = f"{create_subsystem_error_prefix}:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{create_subsystem_error_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + self.logger.error(create_subsystem_error_prefix) + return pb2.req_status(status=errno.EINVAL, error_message=create_subsystem_error_prefix) if context: # Update gateway state try: json_req = json_format.MessageToJson( request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_subsystem(request.subsystem_nqn, - json_req) + self.gateway_state.add_subsystem(request.subsystem_nqn, json_req) except Exception as ex: - self.logger.error(f"Error persisting create_subsystem" - f" {request.subsystem_nqn}: {ex}") - raise + errmsg = f"Error persisting subsystem {request.subsystem_nqn}:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - return pb2.req_status(status=ret) + return pb2.req_status(status=0, error_message=os.strerror(0)) def create_subsystem(self, request, context=None): return self.execute_grpc_function(self.create_subsystem_safe, request, context) + def get_subsystem_namespaces(self, nqn) -> list: + ns_list = [] + local_state_dict = self.gateway_state.local.get_state() + for key, val in local_state_dict.items(): + if not key.startswith(self.gateway_state.local.NAMESPACE_PREFIX): + continue + try: + ns = json.loads(val) + if ns["subsystem_nqn"] == nqn: + nsid = ns["nsid"] + ns_list.append(nsid) + except Exception as ex: + self.logger.error(f"Got exception trying to get subsystem {nqn} namespaces:\n{ex}") + pass + + return ns_list + + def subsystem_has_listeners(self, nqn) -> bool: + local_state_dict = self.gateway_state.local.get_state() + for key, val in local_state_dict.items(): + if not key.startswith(self.gateway_state.local.LISTENER_PREFIX): + continue + try: + lsnr = json.loads(val) + if lsnr["nqn"] == nqn: + return True + except Exception as ex: + self.logger.error(f"Got exception trying to get subsystem {nqn} listener:\n{ex}") + pass + + return False + + def remove_subsystem_from_state(self, nqn, context): + if not context: + return pb2.req_status(status=0, error_message=os.strerror(0)) + + # Update gateway state + try: + self.gateway_state.remove_subsystem(nqn) + except Exception as ex: + errmsg = f"Error persisting deletion of subsystem {nqn}:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + return pb2.req_status(status=0, error_message=os.strerror(0)) + def delete_subsystem_safe(self, request, context=None): """Deletes a subsystem.""" - self.logger.info( - f"Received request to delete subsystem {request.subsystem_nqn}, context: {context}") - - if self.is_discovery_nqn(request.subsystem_nqn): - raise Exception(f"Can't delete a discovery subsystem") + delete_subsystem_error_prefix = f"Failure deleting subsystem {request.subsystem_nqn}" + ret = False with self.omap_lock(context=context): try: ret = rpc_nvmf.nvmf_delete_subsystem( @@ -416,37 +522,81 @@ def delete_subsystem_safe(self, request, context=None): ) self.logger.info(f"delete_subsystem {request.subsystem_nqn}: {ret}") except Exception as ex: - self.logger.error(f"delete_subsystem failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + errmsg = f"{delete_subsystem_error_prefix}:\n{ex}" + self.logger.error(errmsg) + self.remove_subsystem_from_state(request.subsystem_nqn, context) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{delete_subsystem_error_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + self.logger.error(delete_subsystem_error_prefix) + self.remove_subsystem_from_state( request.subsystem_nqn, context) + return pb2.req_status(status=errno.EINVAL, error_message=delete_subsystem_error_prefix) + + return self.remove_subsystem_from_state(request.subsystem_nqn, context) - if context: - # Update gateway state - try: - self.gateway_state.remove_subsystem(request.subsystem_nqn) - except Exception as ex: - self.logger.error(f"Error persisting delete_subsystem" - f" {request.subsystem_nqn}: {ex}") - raise + def delete_subsystem(self, request, context=None): + """Deletes a subsystem.""" - return pb2.req_status(status=ret) + delete_subsystem_error_prefix = f"Failure deleting subsystem {request.subsystem_nqn}" + self.logger.info(f"Received request to delete subsystem {request.subsystem_nqn}, context: {context}") - def delete_subsystem(self, request, context=None): + if self.is_discovery_nqn(request.subsystem_nqn): + errmsg = f"{delete_subsystem_error_prefix}: Can't delete a discovery subsystem" + ret = pb2.req_status(status = errno.EINVAL, error_message = errmsg) + self.logger.error(f"{errmsg}") + return ret + + ns_list = [] + if context: + if self.subsystem_has_listeners(request.subsystem_nqn): + self.logger.warning(f"About to delete subsystem {request.subsystem_nqn} which has a listener defined") + ns_list = self.get_subsystem_namespaces(request.subsystem_nqn) + + # We found a namespace still using this subsystem and --force wasn't used fail with EBUSY + if not request.force and len(ns_list) > 0: + errmsg = f"{delete_subsystem_error_prefix}: Namespace {ns_list[0]} is still using the subsystem. Either remove it or use the '--force' command line option" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EBUSY, error_message=errmsg) + + for nsid in ns_list: + # We found a namespace still using this subsystem and --force was used so we will try to remove the namespace + self.logger.warning(f"Will remove namespace {nsid} from {request.subsystem_nqn}") + ret = self.namespace_delete(pb2.namespace_delete_req(subsystem_nqn=request.subsystem_nqn, nsid=nsid), context) + if ret.status == 0: + self.logger.info(f"Automatically removed namespace {nsid} from {request.subsystem_nqn}") + else: + self.logger.error(f"Failure removing namespace {nsid} from {request.subsystem_nqn}:\n{ret.error_message}") + self.logger.warning(f"Will continue deleting {request.subsystem_nqn} anyway") return self.execute_grpc_function(self.delete_subsystem_safe, request, context) - def add_namespace_safe(self, request, context=None): + def create_namespace_safe(self, request, context=None): """Adds a namespace to a subsystem.""" - - self.logger.info(f"Received request to add {request.bdev_name} to" - f" {request.subsystem_nqn}, context: {context}") + + nsid_msg = "" + if request.nsid != 0: + nsid_msg = f" using NSID {request.nsid}" + add_namespace_error_prefix = f"Failure adding namespace{nsid_msg} to {request.subsystem_nqn}" + + nsid_msg = "" + if request.nsid != 0: + nsid_msg = f" using NSID {request.nsid}" + self.logger.info(f"Received request to add {request.bdev_name} to {request.subsystem_nqn} with ANA group id {request.anagrpid}{nsid_msg}, context: {context}") if request.anagrpid > MAX_ANA_GROUPS: - raise Exception(f"Error group ID {request.anagrpid} is more than configured maximum {MAX_ANA_GROUPS}") + errmsg = f"{add_namespace_error_prefix}: Group ID {request.anagrpid} is bigger than configured maximum {MAX_ANA_GROUPS}" + self.logger.error(errmsg) + return pb2.nsid_status(status=errno.EINVAL, error_message=errmsg) if self.is_discovery_nqn(request.subsystem_nqn): - raise Exception(f"Can't add a namespace to a discovery subsystem") + errmsg = f"{add_namespace_error_prefix}: Can't add namespaces to a discovery subsystem" + self.logger.error(errmsg) + return pb2.nsid_status(status=errno.EINVAL, error_message=errmsg) with self.omap_lock(context=context): try: @@ -456,14 +606,23 @@ def add_namespace_safe(self, request, context=None): bdev_name=request.bdev_name, nsid=request.nsid, anagrpid=request.anagrpid, + uuid=request.uuid, ) self.logger.info(f"add_namespace: {nsid}") except Exception as ex: - self.logger.error(f"add_namespace failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.nsid_status() + errmsg = f"{add_namespace_error_prefix}:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{add_namespace_error_prefix}: {resp['message']}" + return pb2.nsid_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not nsid: + self.logger.error(add_namespace_error_prefix) + return pb2.nsid_status(status=errno.EINVAL, error_message=add_namespace_error_prefix) if context: # Update gateway state @@ -472,26 +631,160 @@ def add_namespace_safe(self, request, context=None): request.nsid = nsid json_req = json_format.MessageToJson( request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_namespace(request.subsystem_nqn, - str(nsid), json_req) + self.gateway_state.add_namespace(request.subsystem_nqn, str(nsid), json_req) except Exception as ex: - self.logger.error( - f"Error persisting add_namespace {nsid}: {ex}") - raise + errmsg = f"Error persisting namespace {nsid} on {request.subsystem_nqn}:\n{ex}" + self.logger.error(errmsg) + return pb2.nsid_status(status=errno.EINVAL, error_message=errmsg) + + return pb2.nsid_status(nsid=nsid, status=0, error_message=os.strerror(0)) + + def create_namespace(self, request, context=None): + return self.execute_grpc_function(self.create_namespace_safe, request, context) + + def find_unique_bdev_name(self) -> str: + with self.rpc_lock: + try: + bdevs = rpc_bdev.bdev_get_bdevs(self.spdk_rpc_client) + except Exception as ex: + self.logger.error(f"Got exception while getting bdevs:\n{ex}") + bdevs = [] + bdev_names = [] + for bdev in bdevs: + try: + bdev_names.append(bdev["name"]) + except Exception as ex: + self.logger.warning(f"Got exception while fteching bdev names:\n{ex}") + pass + random.seed() + randser = random.randint(1, 99999999999999) + bdev_name = f"bdev_{randser}" + while bdev_name in bdev_names: + randser = random.randint(1, 99999999999999) + bdev_name = f"bdev_{randser}" + return bdev_name + + def namespace_add(self, request, context=None): + """Adds a namespace to a subsystem.""" - return pb2.nsid_status(nsid=nsid, status=True) + nsid_msg = "" + if request.nsid: + nsid_msg = f"using NSID {request.nsid} " - def add_namespace(self, request, context=None): - return self.execute_grpc_function(self.add_namespace_safe, request, context) + self.logger.info(f"Received request to add a namespace {nsid_msg}to {request.subsystem_nqn}, context: {context}") + + # We shouldn't get here from an update() + assert context != None + + if not request.uuid: + request.uuid = str(uuid.uuid4()) + bdev_name = self.find_unique_bdev_name() + + create_bdev_req = pb2.create_bdev_req(bdev_name=bdev_name, + rbd_pool_name=request.rbd_pool_name, + rbd_image_name=request.rbd_image_name, + block_size=request.block_size, + uuid=request.uuid) + + ret_bdev = self.create_bdev(create_bdev_req, context) + if ret_bdev.status != 0: + errmsg = f"Failure adding namespace {nsid_msg}to {request.subsystem_nqn}: {ret_bdev.error_message}" + self.logger.error(errmsg) + # Delete the bdev in case the error was from the OMAP and the bdev was created + if "Error persisting bdev" in ret_bdev.error_message: + delete_bdev_req = pb2.delete_bdev_req(bdev_name=bdev_name, force=False) + try: + ret_del = self.delete_bdev(delete_bdev_req, context) + self.logger.info(f"delete_bdev({bdev_name}): {ret_del.status}") + except Exception as ex: + self.logger.warning(f"Got exception while trying to delete bdev {bdev_name}:\n{ex}") + return pb2.nsid_status(status=ret_bdev.status, error_message=errmsg) + + create_namesapce_req = pb2.create_namespace_req(subsystem_nqn=request.subsystem_nqn, + bdev_name=bdev_name, + nsid=request.nsid, + anagrpid=request.anagrpid, + uuid=request.uuid) + ret_ns = self.create_namespace(create_namesapce_req, context) + if ret_ns.status != 0: + delete_bdev_req = pb2.delete_bdev_req(bdev_name=bdev_name, force=True) + try: + ret_del = self.delete_bdev(delete_bdev_req, context) + if ret_del.status != 0: + self.logger.warning(f"Failure {ret_del.status} deleting bdev {bdev_name}: {ret_del.error_message}") + except Exception as ex: + self.logger.warning(f"Got exception while trying to delete bdev {bdev_name}:\n{ex}") + errmsg = f"Failure adding namespace {nsid_msg}to {request.subsystem_nqn}:{ret_ns.error_message}" + self.logger.error(errmsg) + return pb2.nsid_status(status=ret_ns.status, error_message=errmsg) + + return pb2.nsid_status(status=0, error_message=os.strerror(0), nsid=ret_ns.nsid) + + def namespace_change_load_balancing_group(self, request, context=None): + """Changes a namespace load balancing group.""" + + self.logger.info(f"Received request to change load balancing group for namespace {request.nsid} in {request.subsystem_nqn} to {request.anagrpid}, context: {context}") + + find_ret = self.find_namespace_bdev_name(request.subsystem_nqn, request.nsid, f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}") + if not find_ret[0]: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}: Can't find namespace" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + try: + uuid = find_ret[0]["uuid"] + except KeyError: + uuid = None + self.logger.warning(f"Can't get UUID value for namespace {request.nsid} in {request.subsystem_nqn}:\n{find_ret[0]}") + bdev_name = find_ret[1] + if not bdev_name: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}: Can't find bdev" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + + del_ns_req = pb2.remove_namespace_req(subsystem_nqn=request.subsystem_nqn, nsid=request.nsid) + ret_del = self.remove_namespace(del_ns_req, context) + if ret_del.status != 0: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}. Can't delete namespace: {ret_del.error_message}" + self.logger.error(errmsg) + return pb2.req_status(status=ret_del.status, error_message=errmsg) + + create_namesapce_req = pb2.create_namespace_req(subsystem_nqn=request.subsystem_nqn, + bdev_name=bdev_name, + nsid=request.nsid, + anagrpid=request.anagrpid, + uuid=uuid) + ret_ns = self.create_namespace(create_namesapce_req, context) + if ret_ns.status != 0: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}:{ret_ns.error_message}" + self.logger.error(errmsg) + return pb2.req_status(status=ret_ns.status, error_message=errmsg) + + return pb2.req_status(status=0, error_message=os.strerror(0)) + + def remove_namespace_from_state(self, nqn, nsid, context): + if not context: + return pb2.req_status(status=0, error_message=os.strerror(0)) + + # Update gateway state + try: + self.gateway_state.remove_namespace(nqn, str(nsid)) + except Exception as ex: + errmsg = f"Error persisting removing of namespace {nsid} from {nqn}:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + return pb2.req_status(status=0, error_message=os.strerror(0)) def remove_namespace_safe(self, request, context=None): """Removes a namespace from a subsystem.""" - self.logger.info(f"Received request to remove nsid {request.nsid} from" + namespace_failure_prefix = f"Failure removing namespace {request.nsid} from {request.subsystem_nqn}" + self.logger.info(f"Received request to remove namespace {request.nsid} from" f" {request.subsystem_nqn}, context: {context}") if self.is_discovery_nqn(request.subsystem_nqn): - raise Exception(f"Can't remove a namespace from a discovery subsystem") + errmsg=f"{namespace_failure_prefix}: Can't remove a namespace from a discovery subsystem" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) with self.omap_lock(context=context): try: @@ -502,26 +795,374 @@ def remove_namespace_safe(self, request, context=None): ) self.logger.info(f"remove_namespace {request.nsid}: {ret}") except Exception as ex: - self.logger.error(f"remove_namespace failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + errmsg = f"{namespace_failure_prefix}:\n{ex}" + self.logger.error(errmsg) + self.remove_namespace_from_state(request.subsystem_nqn, request.nsid, context) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{namespace_failure_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + self.logger.error(namespace_failure_prefix) + self.remove_namespace_from_state(request.subsystem_nqn, request.nsid, context) + return pb2.req_status(status=errno.EINVAL, error_message=namespace_failure_prefix) + + return self.remove_namespace_from_state(request.subsystem_nqn, request.nsid, context) - if context: - # Update gateway state + def remove_namespace(self, request, context=None): + return self.execute_grpc_function(self.remove_namespace_safe, request, context) + + def get_bdev_info(self, bdev_name): + """Get bdev info""" + + ret_bdev = None + with self.rpc_lock: + try: + bdevs = rpc_bdev.bdev_get_bdevs(self.spdk_rpc_client, name=bdev_name) + if (len(bdevs) > 1): + self.logger.warning(f"Got {len(bdevs)} bdevs for bdev name {bdev_name}, will use the first one") + ret_bdev = bdevs[0] + except Exception as ex: + self.logger.error(f"Got exception while getting bdev {bdev_name} info:\n{ex}") + + return ret_bdev + + def list_namespaces(self, request, context): + """List namespaces.""" + + if request.nsid == None or request.nsid == 0: + nsid_msg = "all namespaces" + else: + nsid_msg = f"namespace {request.nsid}" + self.logger.info(f"Received request to list {nsid_msg} for {request.subsystem}, context: {context}") + + with self.rpc_lock: + try: + ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + self.logger.info(f"list_namespaces: {ret}") + except Exception as ex: + errmsg = f"Failure listing namespaces:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure listing namespaces: {resp['message']}" + return pb2.namespaces_info(status=status, error_message=errmsg, subsystem_nqn=request.subsystem, namespaces=[]) + + namespaces = [] + for s in ret: + try: + if s["nqn"] != request.subsystem: + self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + continue try: - self.gateway_state.remove_namespace(request.subsystem_nqn, - str(request.nsid)) - except Exception as ex: - self.logger.error( - f"Error persisting remove_namespace {request.nsid}: {ex}") - raise + ns_list = s["namespaces"] + except Exception: + ns_list = [] + pass + for n in ns_list: + if request.nsid and request.nsid != n["nsid"]: + self.logger.debug(f'Filter out namespace {n["nsid"]} which is different than requested nsid {request.nsid}') + continue + if request.uuid and request.uuid != n["uuid"]: + self.logger.debug(f'Filter out namespace with UUID {n["uuid"]} which is different than requested UUID {request.uuid}') + continue + bdev_name = n["bdev_name"] + ns_bdev = self.get_bdev_info(bdev_name) + lb_group = 0 + try: + lb_group = n["anagrpid"] + except KeyError: + pass + one_ns = pb2.namespace(nsid = n["nsid"], + bdev_name = n["bdev_name"], + uuid = n["uuid"], + load_balancing_group = lb_group) + if ns_bdev == None: + self.logger.warning(f"Can't find namespace's bdev {bdev_name}, will not list bdev's information") + else: + try: + drv_specific_info = ns_bdev["driver_specific"] + rbd_info = drv_specific_info["rbd"] + one_ns.rbd_image_name = rbd_info["rbd_name"] + one_ns.rbd_pool_name = rbd_info["pool_name"] + one_ns.block_size = ns_bdev["block_size"] + one_ns.rbd_image_size = ns_bdev["block_size"] * ns_bdev["num_blocks"] + assigned_limits = ns_bdev["assigned_rate_limits"] + one_ns.rw_ios_per_second=assigned_limits["rw_ios_per_sec"] + one_ns.rw_mbytes_per_second=assigned_limits["rw_mbytes_per_sec"] + one_ns.r_mbytes_per_second=assigned_limits["r_mbytes_per_sec"] + one_ns.w_mbytes_per_second=assigned_limits["w_mbytes_per_sec"] + except KeyError as err: + self.logger.warning(f"Key {err} is not found, will not list bdev's information") + pass + except Exception: + self.logger.exception(f"{ns_bdev=} parse error: ") + pass + namespaces.append(one_ns) + break + except Exception: + self.logger.exception(f"{s=} parse error: ") + pass - return pb2.req_status(status=ret) + return pb2.namespaces_info(status = 0, error_message = os.strerror(0), subsystem_nqn=request.subsystem, namespaces=namespaces) - def remove_namespace(self, request, context=None): - return self.execute_grpc_function(self.remove_namespace_safe, request, context) + def namespace_get_io_stats(self, request, context): + """Get namespace's IO stats.""" + + self.logger.info(f"Received request to get IO stats for namespace {request.nsid} on {request.subsystem_nqn}, context: {context}") + + find_ret = self.find_namespace_bdev_name(request.subsystem_nqn, request.nsid, "Failure getting namespace's IO stats") + if not find_ret[0]: + errmsg = f"Failure getting namespace's IO stats: Can't find namespace" + self.logger.error(errmsg) + return pb2.namespace_io_stats_info(status=errno.ENODEV, error_message=errmsg) + bdev_name = find_ret[1] + if not bdev_name: + errmsg = f"Failure getting namespace's IO stats: Can't find bdev" + self.logger.error(errmsg) + return pb2.namespace_io_stats_info(status=errno.ENODEV, error_message=errmsg) + + with self.rpc_lock: + try: + ret = rpc_bdev.bdev_get_iostat( + self.spdk_rpc_client, + name=bdev_name, + ) + self.logger.info(f"get_bdev_iostat {bdev_name}: {ret}") + except Exception as ex: + errmsg = f"Failure getting namespace's IO stats:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure getting namespace's IO stats: {resp['message']}" + return pb2.namespace_io_stats_info(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + errmsg = f"Failure getting namespace's IO stats" + self.logger.error(errmsg) + return pb2.namespace_io_stats_info(status=errno.EINVAL, error_message=errmsg) + + exmsg = "" + try: + bdevs = ret["bdevs"] + if not bdevs: + return pb2.namespace_io_stats_info(status=errno.EINVAL, + error_message=f"Failure getting namespace's IO stats: No bdevs found") + if len(bdevs) > 1: + self.logger.warning(f"More than one bdev returned for namespace, will use the first one") + bdev = bdevs[0] + io_stats = pb2.namespace_io_stats_info(status=0, + error_message=os.strerror(0), + subsystem_nqn=request.subsystem_nqn, + bdev_name=bdev_name, + tick_rate=ret["tick_rate"], + ticks=ret["ticks"], + bytes_read=bdev["bytes_read"], + num_read_ops=bdev["num_read_ops"], + bytes_written=bdev["bytes_written"], + num_write_ops=bdev["num_write_ops"], + bytes_unmapped=bdev["bytes_unmapped"], + num_unmap_ops=bdev["num_unmap_ops"], + read_latency_ticks=bdev["read_latency_ticks"], + max_read_latency_ticks=bdev["max_read_latency_ticks"], + min_read_latency_ticks=bdev["min_read_latency_ticks"], + write_latency_ticks=bdev["write_latency_ticks"], + max_write_latency_ticks=bdev["max_write_latency_ticks"], + min_write_latency_ticks=bdev["min_write_latency_ticks"], + unmap_latency_ticks=bdev["unmap_latency_ticks"], + max_unmap_latency_ticks=bdev["max_unmap_latency_ticks"], + min_unmap_latency_ticks=bdev["min_unmap_latency_ticks"], + copy_latency_ticks=bdev["copy_latency_ticks"], + max_copy_latency_ticks=bdev["max_copy_latency_ticks"], + min_copy_latency_ticks=bdev["min_copy_latency_ticks"], + io_error=bdev["io_error"]) + return io_stats + except Exception as ex: + self.logger.exception(f"{s=} parse error: ") + exmsg = str(ex) + pass + + return pb2.namespace_io_stats_info(status=errno.EINVAL, + error_message=f"Failure getting namespace's IO stats: Error parsing returned stats:\n{exmsg}") + + def namespace_set_qos_limits(self, request, context): + """Set namespace's qos limits.""" + + limits_to_set = "" + if request.rw_ios_per_second_is_set: + limits_to_set += f" R/W IOs per second: {request.rw_ios_per_second}" + if request.rw_mbytes_per_second_is_set: + limits_to_set += f" R/W megabytes per second: {request.rw_mbytes_per_second}" + if request.r_mbytes_per_second_is_set: + limits_to_set += f" Read megabytes per second: {request.r_mbytes_per_second}" + if request.w_mbytes_per_second_is_set: + limits_to_set += f" Write megabytes per second: {request.w_mbytes_per_second}" + self.logger.info(f"Received request to set QOS limits for namespace {request.nsid} on {request.subsystem_nqn}, {limits_to_set}, context: {context}") + + find_ret = self.find_namespace_bdev_name(request.subsystem_nqn, request.nsid, "Failure setting namespace's QOS limits") + if not find_ret[0]: + errmsg = f"Failure setting namespace's QOS limits: Can't find namespace" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + bdev_name = find_ret[1] + if not bdev_name: + errmsg = f"Failure setting namespace's QOS limits: Can't find bdev" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + + ret=[False, False, False, False] + with self.rpc_lock: + try: + if request.rw_ios_per_second_is_set: + ret[0] = rpc_bdev.bdev_set_qos_limit( + self.spdk_rpc_client, + name=bdev_name, + rw_ios_per_sec=request.rw_ios_per_second) + else: + ret[0] = True + + if request.rw_mbytes_per_second_is_set: + ret[1] = rpc_bdev.bdev_set_qos_limit( + self.spdk_rpc_client, + name=bdev_name, + rw_mbytes_per_sec=request.rw_mbytes_per_second) + else: + ret[1] = True + + if request.r_mbytes_per_second_is_set: + ret[2] = rpc_bdev.bdev_set_qos_limit( + self.spdk_rpc_client, + name=bdev_name, + r_mbytes_per_sec=request.r_mbytes_per_second) + else: + ret[2] = True + + if request.w_mbytes_per_second_is_set: + ret[3] = rpc_bdev.bdev_set_qos_limit( + self.spdk_rpc_client, + name=bdev_name, + w_mbytes_per_sec=request.w_mbytes_per_second) + else: + ret[3] = True + + self.logger.info(f"bdev_set_qos_limit {bdev_name}: {ret}") + except Exception as ex: + errmsg = f"Failure setting namespace's QOS limits:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure setting namespace's QOS limits: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not all(ret): + errmsg = f"Failure setting namespace's QOS limits" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + + return pb2.req_status(status=0, error_message=os.strerror(0)) + + def find_namespace_bdev_name(self, nqn, nsid, err_prefix): + with self.rpc_lock: + try: + ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=nqn) + self.logger.info(f"find_namespace_bdev_name: {ret}") + except Exception as ex: + errmsg = f"{err_prefix}:\n{ex}" + self.logger.error(errmsg) + return (False, None) + + if not ret: + return (False, None) + + bdev_name = None + found_ns = None + for s in ret: + try: + if s["nqn"] != nqn: + self.logger.warning(f'Got subsystem {s["nqn"]} instead of {nqn}, ignore') + continue + try: + ns_list = s["namespaces"] + except Exception: + ns_list = [] + pass + for n in ns_list: + if nsid != n["nsid"]: + continue + found_ns = n + bdev_name = n["bdev_name"] + break + except Exception: + self.logger.exception(f"{s=} parse error: ") + pass + + return (found_ns, bdev_name) + + def namespace_resize(self, request, context): + """Resize a namespace.""" + + self.logger.info(f"Received request to resize namespace {request.nsid} on {request.subsystem_nqn} to {request.new_size} MiB, context: {context}") + + find_ret = self.find_namespace_bdev_name(request.subsystem_nqn, request.nsid, "Failure resizing namespace") + if not find_ret[0]: + errmsg = f"Failure resizing namespace: Can't find namespace" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + bdev_name = find_ret[1] + if not bdev_name: + errmsg = f"Failure resizing namespace: Can't find bdev" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + + ret = self.resize_bdev(bdev_name, request.new_size) + + if ret.status == 0: + errmsg = os.strerror(0) + else: + errmsg = f"Failure resizing namespace: {ret.error_message}" + self.logger.error(errmsg) + + return pb2.req_status(status=ret.status, error_message=errmsg) + + def namespace_delete(self, request, context): + """Delete a namespace.""" + + self.logger.info(f"Received request to delete namespace {request.nsid} from {request.subsystem_nqn}, context: {context}") + + find_ret = self.find_namespace_bdev_name(request.subsystem_nqn, request.nsid, "Failure deleting namespace") + if not find_ret[0]: + errmsg = f"Failure deleting namespace: Can't find namespace" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + bdev_name = find_ret[1] + if not bdev_name: + self.logger.warning(f"Can't find namespace's bdev name, will try to delete namespace anyway") + + remove_namespace_req = pb2.remove_namespace_req(subsystem_nqn=request.subsystem_nqn, nsid=request.nsid) + ret = self.remove_namespace(remove_namespace_req, context) + if ret.status != 0 or not bdev_name: + return ret + + delete_bdev_req = pb2.delete_bdev_req(bdev_name=bdev_name, force=False) + ret_del = self.delete_bdev(delete_bdev_req, context) + if ret_del.status != 0: + errmsg = f"Failure deleting namespace {request.nsid} from {request.subsystem_nqn}: {ret_del.error_message}" + self.logger.error(errmsg) + return pb2.nsid_status(status=ret_del.status, error_message=errmsg) + + return pb2.req_status(status=0, error_message=os.strerror(0)) def matching_host_exists(self, context, subsys_nqn, host_nqn) -> bool: if not context: @@ -536,34 +1177,37 @@ def matching_host_exists(self, context, subsys_nqn, host_nqn) -> bool: def add_host_safe(self, request, context=None): """Adds a host to a subsystem.""" + all_host_failure_prefix=f"Failure allowing open host access to {request.subsystem_nqn}" + host_failure_prefix=f"Failure adding host {request.host_nqn} to {request.subsystem_nqn}" + if self.is_discovery_nqn(request.subsystem_nqn): - raise Exception(f"Can't allow a host to a discovery subsystem") + if request.host_nqn == "*": + errmsg=f"{all_host_failure_prefix}: Can't allow host access to a discovery subsystem" + else: + errmsg=f"{host_failure_prefix}: Can't add host to a discovery subsystem" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if self.is_discovery_nqn(request.host_nqn): - raise Exception(f"Can't use a discovery NQN as host NQN") + errmsg=f"{host_failure_prefix}: Can't use a discovery NQN as host's" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) with self.omap_lock(context=context): try: host_already_exist = self.matching_host_exists(context, request.subsystem_nqn, request.host_nqn) if host_already_exist: if request.host_nqn == "*": - self.logger.error(f"All hosts already allowed to {request.subsystem_nqn}") - req = {"subsystem_nqn": request.subsystem_nqn, "host_nqn": request.host_nqn, - "method": "nvmf_subsystem_allow_any_host", "req_id": 0} - ret = {"code": -errno.EEXIST, "message": f"All hosts already allowed to {request.subsystem_nqn}"} + errmsg = f"{all_host_failure_prefix}: Open host access is already allowed" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EEXIST, error_message=errmsg) else: - self.logger.error(f"Host {request.host_nqn} already added to {request.subsystem_nqn}") - req = {"subsystem_nqn": request.subsystem_nqn, "host_nqn": request.host_nqn, - "method": "nvmf_subsystem_add_host", "req_id": 0} - ret = {"code": -errno.EEXIST, "message": f"Host {request.host_nqn} already added to {request.subsystem_nqn}"} - msg = "\n".join(["request:", "%s" % json.dumps(req, indent=2), - "Got JSON-RPC error response", - "response:", - json.dumps(ret, indent=2)]) - raise Exception(msg) + errmsg = f"{host_failure_prefix}: Host is already added" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EEXIST, error_message=errmsg) + if request.host_nqn == "*": # Allow any host access to subsystem - self.logger.info(f"Received request to allow any host to" - f" {request.subsystem_nqn}, context: {context}") + self.logger.info(f"Received request to allow any host access for {request.subsystem_nqn}, context: {context}") ret = rpc_nvmf.nvmf_subsystem_allow_any_host( self.spdk_rpc_client, nqn=request.subsystem_nqn, @@ -572,8 +1216,7 @@ def add_host_safe(self, request, context=None): self.logger.info(f"add_host *: {ret}") else: # Allow single host access to subsystem self.logger.info( - f"Received request to add host {request.host_nqn} to" - f" {request.subsystem_nqn}, context: {context}") + f"Received request to add host {request.host_nqn} to {request.subsystem_nqn}, context: {context}") ret = rpc_nvmf.nvmf_subsystem_add_host( self.spdk_rpc_client, nqn=request.subsystem_nqn, @@ -581,11 +1224,29 @@ def add_host_safe(self, request, context=None): ) self.logger.info(f"add_host {request.host_nqn}: {ret}") except Exception as ex: - self.logger.error(f"add_host failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + if request.host_nqn == "*": + errmsg = f"{all_host_failure_prefix}:\n{ex}" + else: + errmsg = f"{host_failure_prefix}:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + if request.host_nqn == "*": + errmsg = f"{all_host_failure_prefix}: {resp['message']}" + else: + errmsg = f"{host_failure_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + if request.host_nqn == "*": + errmsg = all_host_failure_prefix + else: + errmsg = host_failure_prefix + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if context: # Update gateway state @@ -595,29 +1256,55 @@ def add_host_safe(self, request, context=None): self.gateway_state.add_host(request.subsystem_nqn, request.host_nqn, json_req) except Exception as ex: - self.logger.error( - f"Error persisting add_host {request.host_nqn}: {ex}") - raise + errmsg = f"Error persisting host {request.host_nqn} access addition:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - return pb2.req_status(status=ret) + return pb2.req_status(status=0, error_message=os.strerror(0)) def add_host(self, request, context=None): return self.execute_grpc_function(self.add_host_safe, request, context) + def remove_host_from_state(self, subsystem_nqn, host_nqn, context): + if not context: + return pb2.req_status(status=0, error_message=os.strerror(0)) + + # Update gateway state + try: + self.gateway_state.remove_host(subsystem_nqn, host_nqn) + except Exception as ex: + errmsg = f"Error persisting host {host_nqn} access removal:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + return pb2.req_status(status=0, error_message=os.strerror(0)) + def remove_host_safe(self, request, context=None): """Removes a host from a subsystem.""" + all_host_failure_prefix=f"Failure disabling open host access to {request.subsystem_nqn}" + host_failure_prefix=f"Failure removing host {request.host_nqn} access from {request.subsystem_nqn}" + if self.is_discovery_nqn(request.subsystem_nqn): - raise Exception(f"Can't remove a host from a discovery subsystem") + if request.host_nqn == "*": + errmsg=f"{all_host_failure_prefix}: Can't disable open host access to a discovery subsystem" + else: + errmsg=f"{host_failure_prefix}: Can't remove host access from a discovery subsystem" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if self.is_discovery_nqn(request.host_nqn): - raise Exception(f"Can't use a discovery NQN as host NQN") + if request.host_nqn == "*": + errmsg=f"{all_host_failure_prefix}: Can't use a discovery NQN as host's" + else: + errmsg=f"{host_failure_prefix}: Can't use a discovery NQN as host's" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) with self.omap_lock(context=context): try: if request.host_nqn == "*": # Disable allow any host access self.logger.info( - f"Received request to disable any host access to" + f"Received request to disable open host access to" f" {request.subsystem_nqn}, context: {context}") ret = rpc_nvmf.nvmf_subsystem_allow_any_host( self.spdk_rpc_client, @@ -627,7 +1314,7 @@ def remove_host_safe(self, request, context=None): self.logger.info(f"remove_host *: {ret}") else: # Remove single host access to subsystem self.logger.info( - f"Received request to remove host_{request.host_nqn} from" + f"Received request to remove host {request.host_nqn} access from" f" {request.subsystem_nqn}, context: {context}") ret = rpc_nvmf.nvmf_subsystem_remove_host( self.spdk_rpc_client, @@ -636,26 +1323,219 @@ def remove_host_safe(self, request, context=None): ) self.logger.info(f"remove_host {request.host_nqn}: {ret}") except Exception as ex: - self.logger.error(f"remove_host failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + if request.host_nqn == "*": + errmsg = f"{all_host_failure_prefix}:\n{ex}" + else: + errmsg = f"{host_failure_prefix}:\n{ex}" + self.logger.error(errmsg) + self.remove_host_from_state(request.subsystem_nqn, request.host_nqn, context) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + if request.host_nqn == "*": + errmsg = f"{all_host_failure_prefix}: {resp['message']}" + else: + errmsg = f"{host_failure_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) - if context: - # Update gateway state - try: - self.gateway_state.remove_host(request.subsystem_nqn, - request.host_nqn) - except Exception as ex: - self.logger.error(f"Error persisting remove_host: {ex}") - raise + # Just in case SPDK failed with no exception + if not ret: + if request.host_nqn == "*": + errmsg = all_host_failure_prefix + else: + errmsg = host_failure_prefix + self.logger.error(errmsg) + self.remove_host_from_state(request.subsystem_nqn, request.host_nqn, context) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - return pb2.req_status(status=ret) + return self.remove_host_from_state(request.subsystem_nqn, request.host_nqn, context) def remove_host(self, request, context=None): return self.execute_grpc_function(self.remove_host_safe, request, context) + def list_hosts_safe(self, request, context): + """List hosts.""" + + self.logger.info(f"Received request to list hosts for {request.subsystem}, context: {context}") + try: + ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + self.logger.info(f"list_hosts: {ret}") + except Exception as ex: + errmsg = f"Failure listing hosts, can't get subsystems:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure listing hosts, can't get subsystems: {resp['message']}" + return pb2.hosts_info(status=status, error_message=errmsg, hosts=[]) + + hosts = [] + allow_any_host = False + for s in ret: + try: + if s["nqn"] != request.subsystem: + self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + continue + try: + allow_any_host = s["allow_any_host"] + host_nqns = s["hosts"] + except Exception: + host_nqns = [] + pass + for h in host_nqns: + one_host = pb2.host(nqn = h["nqn"]) + hosts.append(one_host) + break + except Exception: + self.logger.exception(f"{s=} parse error: ") + pass + + return pb2.hosts_info(status = 0, error_message = os.strerror(0), allow_any_host=allow_any_host, + subsystem_nqn=request.subsystem, hosts=hosts) + + def list_hosts(self, request, context): + with self.rpc_lock: + return self.list_hosts_safe(request, context) + + def list_connections_safe(self, request, context): + """List connections.""" + + self.logger.info(f"Received request to list connections for {request.subsystem}, context: {context}") + try: + qpair_ret = rpc_nvmf.nvmf_subsystem_get_qpairs(self.spdk_rpc_client, nqn=request.subsystem) + self.logger.info(f"list_connections get_qpairs: {qpair_ret}") + except Exception as ex: + errmsg = f"Failure listing connections, can't get qpairs:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure listing connections, can't get qpairs: {resp['message']}" + return pb2.connections_info(status=status, error_message=errmsg, connections=[]) + + try: + ctrl_ret = rpc_nvmf.nvmf_subsystem_get_controllers(self.spdk_rpc_client, nqn=request.subsystem) + self.logger.info(f"list_connections get_controllers: {ctrl_ret}") + except Exception as ex: + errmsg = f"Failure listing connections, can't get controllers:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure listing connections, can't get controllers: {resp['message']}" + return pb2.bconnections_info(status=status, error_message=errmsg, connections=[]) + + try: + subsys_ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + self.logger.info(f"list_connections subsystems: {subsys_ret}") + except Exception as ex: + errmsg = f"Failure listing connections, can't get subsystems:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure listing connections, can't get subsystems: {resp['message']}" + return pb2.connections_info(status=status, error_message=errmsg, connections=[]) + + connections = [] + host_nqns = [] + for s in subsys_ret: + try: + if s["nqn"] != request.subsystem: + self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + continue + try: + subsys_hosts = s["hosts"] + except Exception: + subsys_hosts = [] + pass + for h in subsys_hosts: + try: + host_nqns.append(h["nqn"]) + except Exception: + pass + break + except Exception: + self.logger.exception(f"{s=} parse error: ") + pass + + for conn in ctrl_ret: + try: + traddr = "" + trsvcid = 0 + adrfam = "" + trtype = "" + hostnqn = conn["hostnqn"] + connected = False + + for qp in qpair_ret: + try: + if qp["cntlid"] != conn["cntlid"]: + continue + if qp["state"] != "active": + continue + addr = qp["listen_address"] + traddr = addr["traddr"] + trsvcid = int(addr["trsvcid"]) + try: + trtype = addr["trtype"].upper() + except Exception: + pass + try: + adrfam = addr["adrfam"].lower() + except Exception: + pass + break + except Exception as ex: + self.logger.warning(f"Got exception while parsing qpair: {qp}:\n{ex}") + pass + one_conn = pb2.connection(nqn=hostnqn, connected=True, + traddr=traddr, trsvcid=trsvcid, trtype=trtype, adrfam=adrfam, + qpairs_count=conn["num_io_qpairs"], controller_id=conn["cntlid"]) + connections.append(one_conn) + host_nqns.remove(hostnqn) + except Exception: + self.logger.exception(f"{s=} parse error: ") + pass + + for nqn in host_nqns: + one_conn = pb2.connection(nqn=nqn, connected=False, traddr="", trsvcid=0, + qpairs_count=-1, controller_id=-1) + connections.append(one_conn) + + return pb2.connections_info(status = 0, error_message = os.strerror(0), + subsystem_nqn=request.subsystem, connections=connections) + + def list_connections(self, request, context): + with self.rpc_lock: + return self.list_connections_safe(request, context) + + def get_subsystem_ha_status(self, nqn) -> bool: + enable_ha = False + state = self.gateway_state.local.get_state() + subsys_str = state.get(GatewayState.build_subsystem_key(nqn)) + if subsys_str: + self.logger.debug(f"value of sub-system: {subsys_str}") + try: + subsys_dict = json.loads(subsys_str) + try: + enable_ha = subsys_dict["enable_ha"] + except KeyError: + enable_ha = False + self.logger.info(f"Subsystem {nqn} enable_ha: {enable_ha}") + except Exception as ex: + self.logger.error(f"Got exception trying to parse subsystem {nqn}:\n{ex}") + enable_ha = False + pass + else: + self.logger.warning(f"Subsystem {nqn} not found") + return enable_ha + def matching_listener_exists(self, context, nqn, gw_name, trtype, traddr, trsvcid) -> bool: if not context: return False @@ -668,27 +1548,37 @@ def matching_listener_exists(self, context, nqn, gw_name, trtype, traddr, trsvci def create_listener_safe(self, request, context=None): """Creates a listener for a subsystem at a given IP/Port.""" + ret = True traddr = GatewayConfig.escape_address_if_ipv6(request.traddr) + create_listener_error_prefix = f"Failure adding {request.nqn} listener at {traddr}:{request.trsvcid}" trtype = GatewayEnumUtils.get_key_from_value(pb2.TransportType, request.trtype) if trtype == None: - raise Exception(f"Unknown transport type {request.trtype}") + errmsg=f"{create_listener_error_prefix}: Unknown transport type {request.trtype}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, request.adrfam) if adrfam == None: - raise Exception(f"Unknown address family {request.adrfam}") + errmsg=f"{create_listener_error_prefix}: Unknown address family {request.adrfam}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) auto_ha_state = GatewayEnumUtils.get_key_from_value(pb2.AutoHAState, request.auto_ha_state) if auto_ha_state == None: - raise Exception(f"Unknown auto HA state {request.auto_ha_state}") + errmsg=f"{create_listener_error_prefix}: Unknown auto HA state {request.auto_ha_state}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) self.logger.info(f"Received request to create {request.gateway_name}" f" {trtype} {adrfam} listener for {request.nqn} at" f" {traddr}:{request.trsvcid}, auto HA state: {auto_ha_state}, context: {context}") if self.is_discovery_nqn(request.nqn): - raise Exception(f"Can't create a listener for a discovery subsystem") + errmsg=f"{create_listener_error_prefix}: Can't create a listener for a discovery subsystem" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) with self.omap_lock(context=context): try: @@ -696,35 +1586,37 @@ def create_listener_safe(self, request, context=None): listener_already_exist = self.matching_listener_exists( context, request.nqn, request.gateway_name, trtype, request.traddr, request.trsvcid) if listener_already_exist: - self.logger.error(f"{request.nqn} already listens on address {request.traddr} port {request.trsvcid}") - req = {"nqn": request.nqn, "trtype": trtype, "traddr": request.traddr, - "gateway_name": request.gateway_name, - "trsvcid": request.trsvcid, "adrfam": adrfam, - "method": "nvmf_subsystem_add_listener", "req_id": 0} - ret = {"code": -errno.EEXIST, "message": f"{request.nqn} already listens on address {request.traddr} port {request.trsvcid}"} - msg = "\n".join(["request:", "%s" % json.dumps(req, indent=2), - "Got JSON-RPC error response", - "response:", - json.dumps(ret, indent=2)]) - raise Exception(msg) + self.logger.error(f"{request.nqn} already listens on address {traddr}:{request.trsvcid}") + return pb2.req_status(status=errno.EEXIST, + error_message=f"{create_listener_error_prefix}: Subsystem already listens on this address") ret = rpc_nvmf.nvmf_subsystem_add_listener( self.spdk_rpc_client, nqn=request.nqn, trtype=trtype, traddr=request.traddr, - trsvcid=request.trsvcid, + trsvcid=str(request.trsvcid), adrfam=adrfam, ) self.logger.info(f"create_listener: {ret}") else: - raise Exception(f"Gateway name must match current gateway" - f" ({self.gateway_name})") + errmsg=f"{create_listener_error_prefix}: Gateway name must match current gateway ({self.gateway_name})" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOENT, + error_message=errmsg) except Exception as ex: - self.logger.error(f"create_listener failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() + errmsg = f"{create_listener_error_prefix}:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{create_listener_error_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + self.logger.error(create_listener_error_prefix) + return pb2.req_status(status=errno.EINVAL, error_message=create_listener_error_prefix) enable_ha = False if auto_ha_state == "AUTO_HA_UNSET": @@ -744,10 +1636,10 @@ def create_listener_safe(self, request, context=None): enable_ha = False self.logger.info(f"enable_ha: {enable_ha}") except Exception as ex: - self.logger.error(f"Got exception trying to parse subsystem {request.nqn}: {ex}") + self.logger.error(f"Got exception trying to parse subsystem {request.nqn}:\n{ex}") pass else: - self.logger.info(f"No subsystem for {request.nqn}") + self.logger.warning(f"No subsystem for {request.nqn}") else: if context != None: self.logger.error(f"auto_ha_state is set to {auto_ha_state} but we are not in an update()") @@ -765,12 +1657,18 @@ def create_listener_safe(self, request, context=None): ana_state="inaccessible", trtype=trtype, traddr=request.traddr, - trsvcid=request.trsvcid, + trsvcid=str(request.trsvcid), adrfam=adrfam, anagrpid=(x+1) ) except Exception as ex: - self.logger.error(f"set_listener_ana_state failed with:\n{ex}") - raise + errmsg=f"{create_listener_error_prefix}: Error setting ANA state:\n{ex}" + self.logger.error(errmsg) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{create_listener_error_prefix}: Error setting ANA state: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) if context: # Update gateway state @@ -782,34 +1680,55 @@ def create_listener_safe(self, request, context=None): trtype, request.traddr, request.trsvcid, json_req) except Exception as ex: - self.logger.error(f"Error persisting add_listener {request.trsvcid}: {ex}") - raise + errmsg = f"Error persisting listener {traddr}:{request.trsvcid}:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - return pb2.req_status(status=ret) + return pb2.req_status(status=0, error_message=os.strerror(0)) def create_listener(self, request, context=None): return self.execute_grpc_function(self.create_listener_safe, request, context) + def remove_listener_from_state(self, nqn, gw_name, trtype, traddr, port, context): + if not context: + return pb2.req_status(status=0, error_message=os.strerror(0)) + + # Update gateway state + try: + self.gateway_state.remove_listener(nqn, gw_name, trtype, traddr, port) + except Exception as ex: + errmsg = f"Error persisting deletion of listener {traddr}:{port} from {nqn}:\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + return pb2.req_status(status=0, error_message=os.strerror(0)) + def delete_listener_safe(self, request, context=None): """Deletes a listener from a subsystem at a given IP/Port.""" ret = True traddr = GatewayConfig.escape_address_if_ipv6(request.traddr) + delete_listener_error_prefix = f"Failure deleting listener {traddr}:{request.trsvcid} from {request.nqn}" trtype = GatewayEnumUtils.get_key_from_value(pb2.TransportType, request.trtype) if trtype == None: - raise Exception(f"Unknown transport type {request.trtype}") + errmsg=f"{delete_listener_error_prefix}: Unknown transport type {request.trtype}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, request.adrfam) if adrfam == None: - raise Exception(f"Unknown address family {request.adrfam}") + errmsg=f"{delete_listener_error_prefix}: Unknown address family {request.adrfam}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) self.logger.info(f"Received request to delete {request.gateway_name}" f" {trtype} listener for {request.nqn} at" f" {traddr}:{request.trsvcid}, context: {context}") if self.is_discovery_nqn(request.nqn): - raise Exception(f"Can't delete a listener from a discovery subsystem") + errmsg=f"{delete_listener_error_prefix}: Can't delete a listener from a discovery subsystem" + self.logger.error(errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) with self.omap_lock(context=context): try: @@ -819,104 +1738,169 @@ def delete_listener_safe(self, request, context=None): nqn=request.nqn, trtype=trtype, traddr=request.traddr, - trsvcid=request.trsvcid, + trsvcid=str(request.trsvcid), adrfam=adrfam, ) self.logger.info(f"delete_listener: {ret}") else: - raise Exception(f"Gateway name must match current gateway" - f" ({self.gateway_name})") + errmsg=f"{delete_listener_error_prefix}: Gateway name must match current gateway ({self.gateway_name})" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOENT, error_message=errmsg) except Exception as ex: - self.logger.error(f"delete_listener failed with: \n {ex}") - if context: - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() - - if context: - # Update gateway state - try: - self.gateway_state.remove_listener(request.nqn, - request.gateway_name, - trtype, - request.traddr, - request.trsvcid) - except Exception as ex: - self.logger.error( - f"Error persisting delete_listener {request.trsvcid}: {ex}") - raise - - return pb2.req_status(status=ret) + errmsg = f"{delete_listener_error_prefix}:\n{ex}" + self.logger.error(errmsg) + self.remove_listener_from_state(request.nqn, request.gateway_name, trtype, + request.traddr, request.trsvcid, context) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"{delete_listener_error_prefix}: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + # Just in case SPDK failed with no exception + if not ret: + self.logger.error(delete_listener_error_prefix) + self.remove_listener_from_state(request.nqn, request.gateway_name, trtype, + request.traddr, request.trsvcid, context) + return pb2.req_status(status=errno.EINVAL, error_message=delete_listener_error_prefix) + + return self.remove_listener_from_state(request.nqn, request.gateway_name, trtype, + request.traddr, request.trsvcid, context) def delete_listener(self, request, context=None): return self.execute_grpc_function(self.delete_listener_safe, request, context) - def get_subsystems_safe(self, request, context): - """Gets subsystems.""" + def list_listeners_safe(self, request, context): + """List listeners.""" - self.logger.info(f"Received request to get subsystems, context: {context}") - subsystems = [] + self.logger.info(f"Received request to list listeners for {request.subsystem}, context: {context}") try: - ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client) - self.logger.info(f"get_subsystems: {ret}") + ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + self.logger.info(f"list_listeners: {ret}") except Exception as ex: - self.logger.error(f"get_subsystems failed with: \n {ex}") - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.subsystems_info() - + errmsg = f"Failure listing listeners, can't get subsystems:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure listing listeners, can't get subsystems: {resp['message']}" + return pb2.listeners_info(status=status, error_message=errmsg, listeners=[]) + + listeners = [] for s in ret: try: - # Need to adjust values to fit enum constants + if s["nqn"] != request.subsystem: + self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + continue try: listen_addrs = s["listen_addresses"] except Exception: listen_addrs = [] pass for addr in listen_addrs: - try: - addr["trtype"] = addr["trtype"].upper() - except Exception: - pass - try: - addr["adrfam"] = addr["adrfam"].lower() - except Exception: - pass + one_listener = pb2.listener_info(gateway_name = self.gateway_name, + trtype = addr["trtype"].upper(), + adrfam = addr["adrfam"].lower(), + traddr = addr["traddr"], + trsvcid = int(addr["trsvcid"])) + listeners.append(one_listener) + break + except Exception: + self.logger.exception(f"{s=} parse error: ") + pass + + return pb2.listeners_info(status = 0, error_message = os.strerror(0), listeners=listeners) + + def list_listeners(self, request, context): + with self.rpc_lock: + return self.list_listeners_safe(request, context) + + def list_subsystems_safe(self, request, context): + """List subsystems.""" + + ser_msg = "" + if request.serial_number: + ser_msg = f" with serial number {request.serial_number}" + if request.subsystem_nqn: + self.logger.info(f"Received request to list subsystem {request.subsystem_nqn}, context: {context}") + else: + self.logger.info(f"Received request to list subsystems{ser_msg}, context: {context}") + + subsystems = [] + try: + if request.subsystem_nqn: + ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem_nqn) + else: + ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client) + self.logger.info(f"list_subsystems: {ret}") + except Exception as ex: + errmsg = f"Failure listing subsystems:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.ENODEV + if resp: + status = resp["code"] + errmsg = f"Failure listing subsystems: {resp['message']}" + return pb2.subsystems_info(status=status, error_message=errmsg, subsystems=[]) + + for s in ret: + try: + if request.serial_number: + if s["serial_number"] != request.serial_number: + continue + if s["subtype"] == "NVMe": + s["namespace_count"] = len(s["namespaces"]) + s["enable_ha"] = self.get_subsystem_ha_status(s["nqn"]) + else: + s["namespace_count"] = 0 + s["enable_ha"] = False # Parse the JSON dictionary into the protobuf message subsystem = pb2.subsystem() - json_format.Parse(json.dumps(s), subsystem) + json_format.Parse(json.dumps(s), subsystem, ignore_unknown_fields=True) subsystems.append(subsystem) except Exception: self.logger.exception(f"{s=} parse error: ") - raise + pass - return pb2.subsystems_info(subsystems=subsystems) + return pb2.subsystems_info(status = 0, error_message = os.strerror(0), subsystems=subsystems) - def get_subsystems(self, request, context): + def list_subsystems(self, request, context): with self.rpc_lock: - return self.get_subsystems_safe(request, context) + return self.list_subsystems_safe(request, context) def get_spdk_nvmf_log_flags_and_level_safe(self, request, context): """Gets spdk nvmf log flags, log level and log print level""" self.logger.info(f"Received request to get SPDK nvmf log flags and level") + log_flags = [] try: nvmf_log_flags = {key: value for key, value in rpc_log.log_get_flags( self.spdk_rpc_client).items() if key.startswith('nvmf')} - spdk_log_level = {'log_level': rpc_log.log_get_level(self.spdk_rpc_client)} - spdk_log_print_level = {'log_print_level': rpc_log.log_get_print_level( - self.spdk_rpc_client)} - flags_log_level = {**nvmf_log_flags, **spdk_log_level, **spdk_log_print_level} + for flag, flagvalue in nvmf_log_flags.items(): + pb2_log_flag = pb2.spdk_log_flag_info(name = flag, enabled = flagvalue) + log_flags.append(pb2_log_flag) + spdk_log_level = rpc_log.log_get_level(self.spdk_rpc_client) + spdk_log_print_level = rpc_log.log_get_print_level(self.spdk_rpc_client) self.logger.info(f"spdk log flags: {nvmf_log_flags}, " f"spdk log level: {spdk_log_level}, " f"spdk log print level: {spdk_log_print_level}") except Exception as ex: - self.logger.error(f"get_spdk_nvmf_log_flags_and_level failed with: \n {ex}") - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.spdk_nvmf_log_flags_and_level_info() + errmsg = f"Failure getting SPDK log levels and nvmf log flags:\n{ex}" + self.logger.error(f"{errmsg}") + resp = self.parse_json_exeption(ex) + status = errno.ENOKEY + if resp: + status = resp["code"] + errmsg = f"Failure getting SPDK log levels and nvmf log flags: {resp['message']}" + return pb2.spdk_nvmf_log_flags_and_level_info(status = status, error_message = errmsg) return pb2.spdk_nvmf_log_flags_and_level_info( - flags_level=json.dumps(flags_log_level)) + nvmf_log_flags=log_flags, + log_level = spdk_log_level, + log_print_level = spdk_log_print_level, + status = 0, + error_message = os.strerror(0)) def get_spdk_nvmf_log_flags_and_level(self, request, context): with self.rpc_lock: @@ -924,45 +1908,63 @@ def get_spdk_nvmf_log_flags_and_level(self, request, context): def set_spdk_nvmf_logs_safe(self, request, context): """Enables spdk nvmf logs""" - self.logger.info(f"Received request to set SPDK nvmf logs") log_level = None print_level = None + ret_log = False + ret_print = False if request.log_level: - try: - log_level = pb2.LogLevel.keys()[request.log_level] - except Exception: - raise Exception(f"Unknown log level {request.log_level}") + log_level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, request.log_level) + if log_level == None: + errmsg=f"Unknown log level {request.log_level}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) if request.print_level: - try: - print_level = pb2.LogLevel.keys()[request.print_level] - except Exception: - raise Exception(f"Unknown print level {request.print_level}") + print_level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, request.print_level) + if print_level == None: + errmsg=f"Unknown print level {request.print_level}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) + + self.logger.info(f"Received request to set SPDK nvmf logs: log_level: {log_level}, print_level: {print_level}") try: nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() \ if key.startswith('nvmf')] ret = [rpc_log.log_set_flag( self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags] - self.logger.info(f"Set SPDK log flags {nvmf_log_flags} to TRUE") + self.logger.info(f"Set SPDK nvmf log flags {nvmf_log_flags} to TRUE: {ret}") if log_level: ret_log = rpc_log.log_set_level(self.spdk_rpc_client, level=log_level) - self.logger.info(f"Set log level to: {log_level}") - ret.append(ret_log) + self.logger.info(f"Set log level to {log_level}: {ret_log}") if print_level: ret_print = rpc_log.log_set_print_level( self.spdk_rpc_client, level=print_level) - self.logger.info(f"Set log print level to: {print_level}") - ret.append(ret_print) + self.logger.info(f"Set log print level to {print_level}: {ret_print}") except Exception as ex: - self.logger.error(f"set_spdk_nvmf_logs failed with:\n{ex}") - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") + errmsg="Failure setting SPDK log levels:\n{ex}" + self.logger.error(f"{errmsg}") for flag in nvmf_log_flags: rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag) - return pb2.req_status() - - return pb2.req_status(status=all(ret)) + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure setting SPDK log levels: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + status = 0 + errmsg = os.strerror(0) + if log_level and not ret_log: + status = errno.EINVAL + errmsg = "Failure setting SPDK log level" + elif print_level and not ret_print: + status = errno.EINVAL + errmsg = "Failure setting SPDK print log level" + elif not all(ret): + status = errno.EINVAL + errmsg = "Failure setting some SPDK nvmf log flags" + return pb2.req_status(status=status, error_message=errmsg) def set_spdk_nvmf_logs(self, request, context): with self.rpc_lock: @@ -980,12 +1982,21 @@ def disable_spdk_nvmf_logs_safe(self, request, context): rpc_log.log_set_print_level(self.spdk_rpc_client, level='INFO')] ret.extend(logs_level) except Exception as ex: - self.logger.error(f"disable_spdk_nvmf_logs failed with: \n {ex}") - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(f"{ex}") - return pb2.req_status() - - return pb2.req_status(status=all(ret)) + self.logger.error(f"disable_spdk_nvmf_logs failed with:\n{ex}") + errmsg = f"Failure in disable SPDK nvmf log flags\n{ex}" + resp = self.parse_json_exeption(ex) + status = errno.EINVAL + if resp: + status = resp["code"] + errmsg = f"Failure in disable SPDK nvmf log flags: {resp['message']}" + return pb2.req_status(status=status, error_message=errmsg) + + status = 0 + errmsg = os.strerror(0) + if not all(ret): + status = errno.EINVAL + errmsg = "Failure in disable SPDK nvmf log flags" + return pb2.req_status(status=status, error_message=errmsg) def disable_spdk_nvmf_logs(self, request, context): with self.rpc_lock: @@ -1006,32 +2017,44 @@ def parse_version(self, version): return None return (v1, v2, v3) - def get_gateway_info(self, request, context): - """Return gateway's info""" + def get_gateway_info_safe(self, request): + """Get gateway's info""" + self.logger.info(f"Received request to get gateway's info") gw_version_string = os.getenv("NVMEOF_VERSION") cli_version_string = request.cli_version addr = self.config.get_with_default("gateway", "addr", "") port = self.config.get_with_default("gateway", "port", "") - ret = pb2.gateway_info(cli_version = request.cli_version, - gateway_version = gw_version_string, - gateway_name = self.gateway_name, - gateway_group = self.gateway_group, - gateway_addr = addr, - gateway_port = port, - status = True) + ret = pb2.gateway_info(version = gw_version_string, + name = self.gateway_name, + group = self.gateway_group, + addr = addr, + port = port, + status = 0, + error_message = os.strerror(0)) cli_ver = self.parse_version(cli_version_string) gw_ver = self.parse_version(gw_version_string) if cli_ver != None and gw_ver != None and cli_ver < gw_ver: - self.logger.error(f"CLI version {cli_version_string} is older than gateway's version {gw_version_string}") - ret.status = False + ret.status = errno.EINVAL + ret.error_message = f"CLI version {cli_version_string} is older than gateway's version {gw_version_string}" + elif not gw_version_string: + ret.status = errno.ENOKEY + ret.error_message = "Gateway's version not found" + elif not gw_ver: + ret.status = errno.EINVAL + ret.error_message = f"Invalid gateway's version {gw_version_string}" if not cli_version_string: - self.logger.error(f"No CLI version specified") - ret.status = False - if not gw_version_string: - self.logger.error(f"Gateway version not found") - ret.status = False - if not cli_ver or not gw_ver: - ret.status = False - self.logger.info(f"Gateway's info:\n{ret}") + self.logger.warning(f"No CLI version specified, can't check version compatibility") + elif not cli_ver: + self.logger.warning(f"Invalid CLI version {cli_version_string}, can't check version compatibility") + if ret.status == 0: + log_func = self.logger.info + else: + log_func = self.logger.error + log_func(f"Gateway's info:\n{ret}") return ret + + def get_gateway_info(self, request, context): + """Get gateway's info""" + with self.rpc_lock: + return self.get_gateway_info_safe(request) diff --git a/control/proto/gateway.proto b/control/proto/gateway.proto index 34c13f15d..9d7efbf17 100644 --- a/control/proto/gateway.proto +++ b/control/proto/gateway.proto @@ -31,7 +31,7 @@ enum AddressFamily { enum LogLevel { DISABLED = 0; ERROR = 1; - WARN = 2; + WARNING = 2; NOTICE = 3; INFO = 4; DEBUG = 5; @@ -44,11 +44,11 @@ enum AutoHAState { } service Gateway { - // Creates a bdev from an RBD image - rpc create_bdev(create_bdev_req) returns (bdev) {} + // Creates a namespace from an RBD image + rpc namespace_add(namespace_add_req) returns (nsid_status) {} - // Resizes a bdev - rpc resize_bdev(resize_bdev_req) returns (req_status) {} + // Creates a bdev from an RBD image + rpc create_bdev(create_bdev_req) returns (bdev_status) {} // Deletes a bdev rpc delete_bdev(delete_bdev_req) returns (req_status) {} @@ -59,42 +59,114 @@ service Gateway { // Deletes a subsystem rpc delete_subsystem(delete_subsystem_req) returns(req_status) {} - // Adds a namespace to a subsystem - rpc add_namespace(add_namespace_req) returns(nsid_status) {} + // Adds a namespace to a subsystem + rpc create_namespace(create_namespace_req) returns(nsid_status) {} // Removes a namespace from a subsystem rpc remove_namespace(remove_namespace_req) returns(req_status) {} + // List namespaces + rpc list_namespaces(list_namespaces_req) returns(namespaces_info) {} + + // Resizes a namespace + rpc namespace_resize(namespace_resize_req) returns (req_status) {} + + // Gets namespace's IO stats + rpc namespace_get_io_stats(namespace_get_io_stats_req) returns (namespace_io_stats_info) {} + + // Sets namespace's qos limits + rpc namespace_set_qos_limits(namespace_set_qos_req) returns (req_status) {} + + // Changes namespace's load balancing group + rpc namespace_change_load_balancing_group(namespace_change_load_balancing_group_req) returns (req_status) {} + + // Deletes a namespace + rpc namespace_delete(namespace_delete_req) returns (req_status) {} + // Adds a host to a subsystem rpc add_host(add_host_req) returns (req_status) {} // Removes a host from a subsystem rpc remove_host(remove_host_req) returns (req_status) {} + // List hosts + rpc list_hosts(list_hosts_req) returns(hosts_info) {} + + // List connections + rpc list_connections(list_connections_req) returns(connections_info) {} + // Creates a listener for a subsystem at a given IP/Port rpc create_listener(create_listener_req) returns(req_status) {} // Deletes a listener from a subsystem at a given IP/Port rpc delete_listener(delete_listener_req) returns(req_status) {} - // Gets subsystems - rpc get_subsystems(get_subsystems_req) returns(subsystems_info) {} + // List listeners + rpc list_listeners(list_listeners_req) returns(listeners_info) {} + + // List subsystems + rpc list_subsystems(list_subsystems_req) returns(subsystems_info) {} // Gets spdk nvmf log flags and level rpc get_spdk_nvmf_log_flags_and_level(get_spdk_nvmf_log_flags_and_level_req) returns(spdk_nvmf_log_flags_and_level_info) {} - // Disables spdk nvmf logs - rpc disable_spdk_nvmf_logs(disable_spdk_nvmf_logs_req) returns(req_status) {} + // Disables spdk nvmf logs + rpc disable_spdk_nvmf_logs(disable_spdk_nvmf_logs_req) returns(req_status) {} // Set spdk nvmf logs rpc set_spdk_nvmf_logs(set_spdk_nvmf_logs_req) returns(req_status) {} - // Set spdk nvmf logs + // Get gateway info rpc get_gateway_info(get_gateway_info_req) returns(gateway_info) {} } // Request messages +message namespace_add_req { + string rbd_pool_name = 1; + string rbd_image_name = 2; + string subsystem_nqn = 3; + optional uint32 nsid = 4; + int32 block_size = 5; + optional string uuid = 6; + optional int32 anagrpid = 7; +} + +message namespace_resize_req { + string subsystem_nqn = 1; + uint32 nsid = 2; + uint32 new_size = 3; +} + +message namespace_get_io_stats_req { + string subsystem_nqn = 1; + uint32 nsid = 2; +} + +message namespace_set_qos_req { + string subsystem_nqn = 1; + uint32 nsid = 2; + bool rw_ios_per_second_is_set = 3; + uint64 rw_ios_per_second = 4; + bool rw_mbytes_per_second_is_set = 5; + uint64 rw_mbytes_per_second = 6; + bool r_mbytes_per_second_is_set = 7; + uint64 r_mbytes_per_second = 8; + bool w_mbytes_per_second_is_set = 9; + uint64 w_mbytes_per_second = 10; +} + +message namespace_change_load_balancing_group_req { + string subsystem_nqn = 1; + uint32 nsid = 2; + int32 anagrpid = 3; +} + +message namespace_delete_req { + string subsystem_nqn = 1; + uint32 nsid = 2; +} + message create_bdev_req { string bdev_name = 1; string rbd_pool_name = 2; @@ -103,11 +175,6 @@ message create_bdev_req { optional string uuid = 5; } -message resize_bdev_req { - string bdev_name = 1; - int32 new_size = 2; -} - message delete_bdev_req { string bdev_name = 1; bool force = 2; @@ -121,15 +188,17 @@ message create_subsystem_req { bool enable_ha = 5; } -message delete_subsystem_req { - string subsystem_nqn = 1; +message create_namespace_req { + string subsystem_nqn = 1; + string bdev_name = 2; + optional uint32 nsid = 3; + optional int32 anagrpid = 4; + optional string uuid = 5; } -message add_namespace_req { +message delete_subsystem_req { string subsystem_nqn = 1; - string bdev_name = 2; - optional uint32 nsid = 3; - optional int32 anagrpid = 4; + bool force = 2; } message remove_namespace_req { @@ -137,6 +206,12 @@ message remove_namespace_req { uint32 nsid = 2; } +message list_namespaces_req { + string subsystem = 1; + optional uint32 nsid = 2; + optional string uuid = 3; +} + message add_host_req { string subsystem_nqn = 1; string host_nqn = 2; @@ -147,14 +222,22 @@ message remove_host_req { string host_nqn = 2; } +message list_hosts_req { + string subsystem = 1; +} + +message list_connections_req { + string subsystem = 1; +} + message create_listener_req { string nqn = 1; string gateway_name = 2; TransportType trtype = 3; AddressFamily adrfam = 4; string traddr = 5; - string trsvcid = 6; - AutoHAState auto_ha_state = 7; + uint32 trsvcid = 6; + optional AutoHAState auto_ha_state = 7; } message delete_listener_req { @@ -163,12 +246,21 @@ message delete_listener_req { TransportType trtype = 3; AddressFamily adrfam = 4; string traddr = 5; - string trsvcid = 6; + uint32 trsvcid = 6; +} + +message list_listeners_req { + string subsystem = 1; } message get_subsystems_req { } +message list_subsystems_req { + optional string subsystem_nqn = 1; + optional string serial_number = 2; +} + message get_spdk_nvmf_log_flags_and_level_req { } @@ -186,69 +278,165 @@ message get_gateway_info_req { // Return messages -message bdev { - string bdev_name = 1; - bool status = 2; +message bdev_status { + int32 status = 1; + string error_message = 2; + string bdev_name = 3; } message req_status { - bool status = 1; + int32 status = 1; + string error_message = 2; } message nsid_status { - uint32 nsid = 1; - bool status = 2; + int32 status = 1; + string error_message = 2; + uint32 nsid = 3; } message subsystems_info { - repeated subsystem subsystems = 1; + int32 status = 1; + string error_message = 2; + repeated subsystem subsystems = 3; } message subsystem { string nqn = 1; - string subtype = 2; - repeated listen_address listen_addresses = 3; - repeated host hosts = 4; - bool allow_any_host = 5; - optional string serial_number = 6; - optional string model_number = 7; - optional uint32 max_namespaces = 8; - optional uint32 min_cntlid = 9; - optional uint32 max_cntlid = 10; - repeated namespace namespaces = 11; + bool enable_ha = 2; + string serial_number = 3; + string model_number = 4; + uint32 min_cntlid = 5; + uint32 max_cntlid = 6; + uint32 namespace_count = 7; + string subtype = 8; } message gateway_info { - string cli_version = 1; - string gateway_version = 2; - string gateway_name = 3; - string gateway_group = 4; - string gateway_addr = 5; - string gateway_port = 6; - bool status = 7; + int32 status = 1; + string error_message = 2; + string version = 3; + string name = 4; + string group = 5; + string addr = 6; + string port = 7; } -message listen_address { - string transport = 1; +message cli_version { + int32 status = 1; + string error_message = 2; + string version = 3; +} + +message gw_version { + int32 status = 1; + string error_message = 2; + string version = 3; +} + +message listener_info { + string gateway_name = 1; TransportType trtype = 2; AddressFamily adrfam = 3; string traddr = 4; - string trsvcid = 5; + uint32 trsvcid = 5; +} + +message listeners_info { + int32 status = 1; + string error_message = 2; + repeated listener_info listeners = 3; } message host { string nqn = 1; } +message hosts_info { + int32 status = 1; + string error_message = 2; + bool allow_any_host = 3; + string subsystem_nqn = 4; + repeated host hosts = 5; +} + +message connection { + string nqn = 1; + string traddr = 2; + uint32 trsvcid = 3; + TransportType trtype = 4; + AddressFamily adrfam = 5; + bool connected = 6; + int32 qpairs_count = 7; + int32 controller_id = 8; +} + +message connections_info { + int32 status = 1; + string error_message = 2; + string subsystem_nqn = 3; + repeated connection connections = 4; +} + message namespace { - uint32 nsid = 1; - string name = 2; - optional string bdev_name = 3; - optional string nguid = 4; - optional string uuid = 5; - optional uint32 anagrpid = 6; + uint32 nsid = 1; + string bdev_name = 2; + string rbd_image_name = 3; + string rbd_pool_name = 4; + uint32 load_balancing_group = 5; + int32 block_size = 6; + int32 rbd_image_size = 7; + string uuid = 8; + uint64 rw_ios_per_second = 9; + uint64 rw_mbytes_per_second = 10; + uint64 r_mbytes_per_second = 11; + uint64 w_mbytes_per_second = 12; +} + +message namespaces_info { + int32 status = 1; + string error_message = 2; + string subsystem_nqn = 3; + repeated namespace namespaces = 4; +} + +message namespace_io_stats_info { + int32 status = 1; + string error_message = 2; + string subsystem_nqn = 3; + string bdev_name = 4; + uint64 tick_rate = 5; + uint64 ticks = 6; + uint64 bytes_read = 7; + uint64 num_read_ops = 8; + uint64 bytes_written = 9; + uint64 num_write_ops = 10; + uint64 bytes_unmapped = 11; + uint64 num_unmap_ops = 12; + uint64 read_latency_ticks = 13; + uint64 max_read_latency_ticks = 14; + uint64 min_read_latency_ticks = 15; + uint64 write_latency_ticks = 16; + uint64 max_write_latency_ticks = 17; + uint64 min_write_latency_ticks = 18; + uint64 unmap_latency_ticks = 19; + uint64 max_unmap_latency_ticks = 20; + uint64 min_unmap_latency_ticks = 21; + uint64 copy_latency_ticks = 22; + uint64 max_copy_latency_ticks = 23; + uint64 min_copy_latency_ticks = 24; + repeated uint32 io_error = 25; +} + +message spdk_log_flag_info { + string name = 1; + bool enabled = 2; } message spdk_nvmf_log_flags_and_level_info { - string flags_level =1; + int32 status = 1; + string error_message = 2; + repeated spdk_log_flag_info nvmf_log_flags = 3; + LogLevel log_level = 4; + LogLevel log_print_level = 5; } diff --git a/control/server.py b/control/server.py index 2a900775e..5e824df17 100644 --- a/control/server.py +++ b/control/server.py @@ -375,8 +375,8 @@ def gateway_rpc_caller(self, requests, is_add_req): self.gateway_rpc.delete_subsystem(req) elif key.startswith(GatewayState.NAMESPACE_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.add_namespace_req()) - self.gateway_rpc.add_namespace(req) + req = json_format.Parse(val, pb2.create_namespace_req()) + self.gateway_rpc.create_namespace(req) else: req = json_format.Parse(val, pb2.remove_namespace_req(), diff --git a/control/state.py b/control/state.py index 36d8c0cee..2ddbffc6a 100644 --- a/control/state.py +++ b/control/state.py @@ -51,8 +51,8 @@ def build_host_key(subsystem_nqn: str, host_nqn) -> str: def build_partial_listener_key(subsystem_nqn: str) -> str: return GatewayState.LISTENER_PREFIX + subsystem_nqn - def build_listener_key(subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: str) -> str: - return GatewayState.build_partial_listener_key(subsystem_nqn) + "_" + gateway + "_" + trtype + "_" + traddr + "_" + trsvcid + def build_listener_key(subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: int) -> str: + return GatewayState.build_partial_listener_key(subsystem_nqn) + "_" + gateway + "_" + trtype + "_" + traddr + "_" + str(trsvcid) @abstractmethod def get_state(self) -> Dict[str, str]: @@ -118,13 +118,13 @@ def remove_host(self, subsystem_nqn: str, host_nqn: str): self._remove_key(key) def add_listener(self, subsystem_nqn: str, gateway: str, trtype: str, - traddr: str, trsvcid: str, val: str): + traddr: str, trsvcid: int, val: str): """Adds a listener to the state data store.""" key = GatewayState.build_listener_key(subsystem_nqn, gateway, trtype, traddr, trsvcid) self._add_key(key, val) def remove_listener(self, subsystem_nqn: str, gateway: str, trtype: str, - traddr: str, trsvcid: str): + traddr: str, trsvcid: int): """Removes a listener from the state data store.""" key = GatewayState.build_listener_key(subsystem_nqn, gateway, trtype, traddr, trsvcid) self._remove_key(key) diff --git a/mk/demo.mk b/mk/demo.mk index c132aa3b4..afde7a318 100644 --- a/mk/demo.mk +++ b/mk/demo.mk @@ -6,16 +6,13 @@ rbd: SVC = ceph rbd: CMD = bash -c "rbd -p $(RBD_POOL) info $(RBD_IMAGE_NAME) || rbd -p $(RBD_POOL) create $(RBD_IMAGE_NAME) --size $(RBD_IMAGE_SIZE)" # demo -# the fist gateway in docker enviroment, hostname defaults to container id +# the first gateway in docker environment, hostname defaults to container id demo: export NVMEOF_HOSTNAME != docker ps -q -f name=$(NVMEOF_CONTAINER_NAME) demo: rbd ## Expose RBD_IMAGE_NAME as NVMe-oF target - $(NVMEOF_CLI) create_bdev --pool $(RBD_POOL) --image $(RBD_IMAGE_NAME) --bdev $(BDEV_NAME) - $(NVMEOF_CLI_IPV6) create_bdev --pool $(RBD_POOL) --image $(RBD_IMAGE_NAME) --bdev $(BDEV_NAME)_ipv6 - $(NVMEOF_CLI) create_subsystem --subnqn $(NQN) - $(NVMEOF_CLI) add_namespace --subnqn $(NQN) --bdev $(BDEV_NAME) - $(NVMEOF_CLI) add_namespace --subnqn $(NQN) --bdev $(BDEV_NAME)_ipv6 - $(NVMEOF_CLI) create_listener --subnqn $(NQN) --gateway-name $(NVMEOF_HOSTNAME) --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT) - $(NVMEOF_CLI_IPV6) create_listener --subnqn $(NQN) --gateway-name $(NVMEOF_HOSTNAME) --traddr '$(NVMEOF_IPV6_ADDRESS)' --trsvcid $(NVMEOF_IO_PORT) --adrfam IPV6 - $(NVMEOF_CLI) add_host --subnqn $(NQN) --host "*" + $(NVMEOF_CLI) subsystem add --subsystem $(NQN) + $(NVMEOF_CLI) namespace add --subsystem $(NQN) --rbd-pool $(RBD_POOL) --rbd-image $(RBD_IMAGE_NAME) + $(NVMEOF_CLI) listener add --subsystem $(NQN) --gateway-name $(NVMEOF_HOSTNAME) --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT) + $(NVMEOF_CLI_IPV6) listener add --subsystem $(NQN) --gateway-name $(NVMEOF_HOSTNAME) --traddr $(NVMEOF_IPV6_ADDRESS) --trsvcid $(NVMEOF_IO_PORT) --adrfam IPV6 + $(NVMEOF_CLI) host add --subsystem $(NQN) --host "*" .PHONY: demo rbd diff --git a/mk/misc.mk b/mk/misc.mk index 34766b3ef..69572a6f4 100644 --- a/mk/misc.mk +++ b/mk/misc.mk @@ -5,6 +5,6 @@ NVMEOF_CLI = $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) run --rm nvmeof-cli --serve NVMEOF_CLI_IPV6 = $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) run --rm nvmeof-cli --server-address $(NVMEOF_IPV6_ADDRESS) --server-port $(NVMEOF_GW_PORT) alias: ## Print bash alias command for the nvmeof-cli. Usage: "eval $(make alias)" - @echo alias nvmeof-cli=\"$(NVMEOF_CLI)\" \; alias nvmeof-cli-ipv6=\'$(NVMEOF_CLI_IPV6)\' + @echo alias cephnvmf=\"$(NVMEOF_CLI)\" \; alias cephnvmf-ipv6=\'$(NVMEOF_CLI_IPV6)\' .PHONY: alias diff --git a/tests/test_cli.py b/tests/test_cli.py index 3d90e7715..0a711f724 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,23 +10,21 @@ image = "mytestdevimage" pool = "rbd" -bdev = "Ceph0" -bdev1 = "Ceph1" -bdev_ipv6 = bdev + "_ipv6" -bdev1_ipv6 = bdev1 + "_ipv6" subsystem = "nqn.2016-06.io.spdk:cnode1" subsystem2 = "nqn.2016-06.io.spdk:cnode2" discovery_nqn = "nqn.2014-08.org.nvmexpress.discovery" serial = "SPDK00000000000001" +uuid = "948878ee-c3b2-4d58-a29b-2cff713fc02d" host_list = ["nqn.2016-06.io.spdk:host1", "*"] nsid = "1" nsid_ipv6 = "3" anagrpid = "2" +anagrpid2 = "4" gateway_name = socket.gethostname() addr = "127.0.0.1" addr_ipv6 = "::1" server_addr_ipv6 = "2001:db8::3" -listener_list = [["-g", gateway_name, "-a", addr, "-s", "5001", "-t", "tCp", "-f", "ipV4"], ["-g", gateway_name, "-a", addr, "-s", "5002"]] +listener_list = [["-g", gateway_name, "-a", addr, "-s", "5001", "-t", "tcp", "-f", "ipv4"], ["-g", gateway_name, "-a", addr, "-s", "5002"]] listener_list_no_port = [["-g", gateway_name, "-a", addr]] listener_list_fc_trtype = [["-g", gateway_name, "-a", addr, "-s", "5010", "--trtype", "FC"]] listener_list_invalid_trtype = [["-g", gateway_name, "-a", addr, "-s", "5011", "--trtype", "JUNK"]] @@ -34,6 +32,7 @@ listener_list_ib_adrfam = [["-g", gateway_name, "-a", addr, "-s", "5014", "--adrfam", "ib"]] listener_list_ipv6 = [["-g", gateway_name, "-a", addr_ipv6, "-s", "5003", "--adrfam", "ipv6"], ["-g", gateway_name, "-a", addr_ipv6, "-s", "5004", "--adrfam", "IPV6"]] listener_list_discovery = [["-n", discovery_nqn, "-g", gateway_name, "-a", addr, "-s", "5012"]] +listener_list_negative_port = [["-g", gateway_name, "-a", addr, "-s", "-2000"]] config = "ceph-nvmeof.conf" @pytest.fixture(scope="module") @@ -60,430 +59,526 @@ def gateway(config): class TestGet: def test_get_subsystems(self, caplog, gateway): caplog.clear() - cli(["get_subsystems"]) - assert "[]" in caplog.text + cli(["subsystem", "list"]) + assert "No subsystems" in caplog.text def test_get_subsystems_ipv6(self, caplog, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "get_subsystems"]) - assert "[]" in caplog.text + cli(["--server-address", server_addr_ipv6, "subsystem", "list"]) + assert "No subsystems" in caplog.text def test_get_gateway_info(self, caplog, gateway): gw, stub = gateway caplog.clear() gw_info_req = pb2.get_gateway_info_req(cli_version="0.0.1") ret = stub.get_gateway_info(gw_info_req) - assert not ret.status + assert ret.status != 0 assert "is older than gateway" in caplog.text caplog.clear() gw_info_req = pb2.get_gateway_info_req() ret = stub.get_gateway_info(gw_info_req) assert "No CLI version specified" in caplog.text - assert not ret.status + assert ret.status == 0 caplog.clear() gw_info_req = pb2.get_gateway_info_req(cli_version="0.0.1.4") ret = stub.get_gateway_info(gw_info_req) assert "Can't parse version" in caplog.text - assert not ret.status + assert "Invalid CLI version" in caplog.text + assert ret.status == 0 caplog.clear() gw_info_req = pb2.get_gateway_info_req(cli_version="0.X.4") ret = stub.get_gateway_info(gw_info_req) assert "Can't parse version" in caplog.text - assert not ret.status + assert "Invalid CLI version" in caplog.text + assert ret.status == 0 caplog.clear() cli_ver = os.getenv("NVMEOF_VERSION") gw.config.config["gateway"]["port"] = "6789" gw.config.config["gateway"]["addr"] = "10.10.10.10" gw_info_req = pb2.get_gateway_info_req(cli_version=cli_ver) ret = stub.get_gateway_info(gw_info_req) - assert ret.status - assert f'cli_version: "{cli_ver}"' in caplog.text - assert f'gateway_version: "{cli_ver}"' in caplog.text - assert 'gateway_port: "6789"' in caplog.text - assert 'gateway_addr: "10.10.10.10"' in caplog.text - assert f'gateway_name: "{gw.gateway_name}"' in caplog.text - -class TestCreate: - def test_create_bdev(self, caplog, gateway): - gw, stub = gateway - bdev_found = False - caplog.clear() - cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev]) - assert f"Created bdev {bdev}: True" in caplog.text - bdev_list = rpc_bdev.bdev_get_bdevs(gw.spdk_rpc_client) - for onedev in bdev_list: - if onedev["name"] == bdev: - bdev_found = True - assert onedev["block_size"] == 512 - break - assert bdev_found - caplog.clear() - bdev_found = False - cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev1, "-s", "1024"]) - assert f"Created bdev {bdev1}: True" in caplog.text - bdev_list = rpc_bdev.bdev_get_bdevs(gw.spdk_rpc_client) - for onedev in bdev_list: - if onedev["name"] == bdev1: - bdev_found = True - assert onedev["block_size"] == 1024 - break - assert bdev_found - - def test_create_bdev_ipv6(self, caplog, gateway): + assert ret.status == 0 + assert f'version: "{cli_ver}"' in caplog.text + assert 'port: "6789"' in caplog.text + assert 'addr: "10.10.10.10"' in caplog.text + assert f'name: "{gw.gateway_name}"' in caplog.text caplog.clear() - cli(["--server-address", server_addr_ipv6, "create_bdev", "-i", image, "-p", pool, "-b", bdev_ipv6]) - assert f"Created bdev {bdev_ipv6}: True" in caplog.text - cli(["--server-address", server_addr_ipv6, "create_bdev", "-i", image, "-p", pool, "-b", bdev1_ipv6]) - assert f"Created bdev {bdev1_ipv6}: True" in caplog.text - - def test_resize_bdev(self, caplog, gateway): - caplog.clear() - bdev_found = False - gw, stub = gateway - cli(["resize_bdev", "-b", bdev, "-s", "20"]) - assert f"Resized bdev {bdev}: True" in caplog.text - bdev_list = rpc_bdev.bdev_get_bdevs(gw.spdk_rpc_client) - for onedev in bdev_list: - if onedev["name"] == bdev: - bdev_found = True - assert onedev["block_size"] == 512 - num_blocks = onedev["num_blocks"] - # Should be 20M now - assert num_blocks * 512 == 20971520 - break - assert bdev_found + cli(["version"]) + assert f"CLI version: {cli_ver}" in caplog.text +class TestCreate: def test_create_subsystem(self, caplog, gateway): caplog.clear() - cli(["create_subsystem", "-n", subsystem]) - assert f"Created subsystem {subsystem}: True" in caplog.text + cli(["subsystem", "add", "--subsystem", subsystem]) + assert f"create_subsystem {subsystem}: True" in caplog.text assert "ana reporting: False" in caplog.text - cli(["get_subsystems"]) - assert serial not in caplog.text + cli(["--format", "json", "subsystem", "list"]) + assert f'"serial_number": "{serial}"' not in caplog.text + assert f'"nqn": "{subsystem}"' in caplog.text caplog.clear() - cli(["create_subsystem", "-n", subsystem2, "-s", serial]) - assert f"Created subsystem {subsystem2}: True" in caplog.text + cli(["subsystem", "add", "--subsystem", subsystem2, "--serial-number", serial]) + assert f"create_subsystem {subsystem2}: True" in caplog.text assert "ana reporting: False" in caplog.text caplog.clear() - cli(["get_subsystems"]) - assert serial in caplog.text + cli(["--format", "json", "subsystem", "list"]) + assert f'"serial_number": "{serial}"' in caplog.text + assert f'"nqn": "{subsystem}"' in caplog.text + assert f'"nqn": "{subsystem2}"' in caplog.text + caplog.clear() + cli(["--format", "json", "subsystem", "list", "--subsystem", subsystem]) + assert f'"nqn": "{subsystem}"' in caplog.text + assert f'"nqn": "{subsystem2}"' not in caplog.text + caplog.clear() + cli(["--format", "json", "subsystem", "list", "--serial-number", serial]) + assert f'"nqn": "{subsystem}"' not in caplog.text + assert f'"nqn": "{subsystem2}"' in caplog.text + caplog.clear() + cli(["subsystem", "list", "--serial-number", "JUNK"]) + assert f"No subsystem with serial number JUNK" in caplog.text + caplog.clear() + cli(["subsystem", "list", "--subsystem", "JUNK"]) + assert f"Failure listing subsystems: No such device" in caplog.text + assert f'"nqn": "JUNK"' in caplog.text + + def test_create_subsystem_with_discovery_nqn(self, caplog, gateway): + caplog.clear() + rc = 0 + try: + cli(["subsystem", "add", "--subsystem", discovery_nqn]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert "Can't add a discovery subsystem" in caplog.text + assert rc == 2 def test_add_namespace(self, caplog, gateway): caplog.clear() - cli(["add_namespace", "-n", subsystem, "-b", bdev]) - assert f"Added namespace 1 to {subsystem}, ANA group id None : True" in caplog.text - cli(["add_namespace", "-n", subsystem, "-b", bdev1]) - assert f"Added namespace 2 to {subsystem}, ANA group id None : True" in caplog.text + cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--uuid", uuid]) + assert f"Adding namespace 1 to nqn.2016-06.io.spdk:cnode1, load balancing group 1: Successful" in caplog.text + caplog.clear() + cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--block-size", "1024"]) + assert f"Adding namespace 2 to nqn.2016-06.io.spdk:cnode1, load balancing group 1: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"load_balancing_group": 0' in caplog.text + assert '"block_size": 512' in caplog.text + assert f'"uuid": "{uuid}"' in caplog.text + assert '"rw_ios_per_second": "0"' in caplog.text + assert '"rw_mbytes_per_second": "0"' in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "2"]) + assert '"load_balancing_group": 0' in caplog.text + assert '"block_size": 1024' in caplog.text + assert f'"uuid": "{uuid}"' not in caplog.text + assert '"rw_ios_per_second": "0"' in caplog.text + assert '"rw_mbytes_per_second": "0"' in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--uuid", uuid]) + assert f'"uuid": "{uuid}"' in caplog.text + caplog.clear() + cli(["namespace", "change_load_balancing_group", "--subsystem", subsystem, "--nsid", nsid, "--load-balancing-group", anagrpid2]) + assert f"Changing load balancing group of namespace {nsid} in {subsystem} to {anagrpid2}: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", nsid]) + assert '"load_balancing_group": 0' in caplog.text def test_add_namespace_ipv6(self, caplog, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "add_namespace", "-n", subsystem, "-b", bdev_ipv6]) - assert f"Added namespace 3 to {subsystem}, ANA group id None : True" in caplog.text - cli(["--server-address", server_addr_ipv6, "add_namespace", "-n", subsystem, "-b", bdev1_ipv6]) - assert f"Added namespace 4 to {subsystem}, ANA group id None : True" in caplog.text + cli(["--server-address", server_addr_ipv6, "namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image]) + assert f"Adding namespace 3 to {subsystem}, load balancing group 1: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "3"]) + assert '"load_balancing_group": 0' in caplog.text + cli(["--server-address", server_addr_ipv6, "namespace", "add", "--subsystem", subsystem, "--nsid", "8", "--rbd-pool", pool, "--rbd-image", image]) + assert f"Adding namespace 8 to {subsystem}, load balancing group 1: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "8"]) + assert '"load_balancing_group": 0' in caplog.text + + def test_resize_namespace(self, caplog, gateway): + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"nsid": 1' in caplog.text + assert '"block_size": 512' in caplog.text + assert '"rbd_image_size": 16777216' in caplog.text + assert f'"uuid": "{uuid}"' in caplog.text + caplog.clear() + cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "1", "--size", "32"]) + assert f"Resizing namespace 1 on {subsystem} to 32 MiB: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"nsid": 1' in caplog.text + assert '"block_size": 512' in caplog.text + assert '"rbd_image_size": 33554432' in caplog.text + assert f'"uuid": "{uuid}"' in caplog.text + assert '"nsid": 2' not in caplog.text + assert '"nsid": 3' not in caplog.text + assert '"nsid": 4' not in caplog.text + assert '"nsid": 8' not in caplog.text + + def test_set_namespace_qos_limits(self, caplog, gateway): + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"nsid": 1' in caplog.text + assert '"rw_ios_per_second": "0"' in caplog.text + assert '"rw_mbytes_per_second": "0"' in caplog.text + assert '"r_mbytes_per_second": "0"' in caplog.text + assert '"w_mbytes_per_second": "0"' in caplog.text + caplog.clear() + cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "1", "--rw-ios-per-second", "2000"]) + assert f"Setting QOS limits of namespace 1 in {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"nsid": 1' in caplog.text + assert '"rw_ios_per_second": "2000"' in caplog.text + assert '"rw_mbytes_per_second": "0"' in caplog.text + assert '"r_mbytes_per_second": "0"' in caplog.text + assert '"w_mbytes_per_second": "0"' in caplog.text + caplog.clear() + cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "1", "--rw-megabytes-per-second", "30"]) + assert f"Setting QOS limits of namespace 1 in {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"nsid": 1' in caplog.text + assert '"rw_ios_per_second": "2000"' in caplog.text + assert '"rw_mbytes_per_second": "30"' in caplog.text + assert '"r_mbytes_per_second": "0"' in caplog.text + assert '"w_mbytes_per_second": "0"' in caplog.text + caplog.clear() + cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "1", + "--r-megabytes-per-second", "15", "--w-megabytes-per-second", "25"]) + assert f"Setting QOS limits of namespace 1 in {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert '"nsid": 1' in caplog.text + assert '"rw_ios_per_second": "2000"' in caplog.text + assert '"rw_mbytes_per_second": "30"' in caplog.text + assert '"r_mbytes_per_second": "15"' in caplog.text + assert '"w_mbytes_per_second": "25"' in caplog.text + + def test_namespace_io_stats(self, caplog, gateway): + caplog.clear() + cli(["--format", "json", "namespace", "get_io_stats", "--subsystem", subsystem, "--nsid", "1"]) + assert f'"status": 0' in caplog.text + assert f'"subsystem_nqn": "{subsystem}"' in caplog.text + assert f'"ticks":' in caplog.text + assert f'"bytes_written":' in caplog.text + assert f'"bytes_read":' in caplog.text + assert f'"max_write_latency_ticks":' in caplog.text + assert f'"io_error":' in caplog.text @pytest.mark.parametrize("host", host_list) def test_add_host(self, caplog, host): caplog.clear() - cli(["add_host", "-n", subsystem, "-t", host]) + cli(["host", "add", "--subsystem", subsystem, "--host", host]) if host == "*": - assert f"Allowed open host access to {subsystem}: True" in caplog.text + assert f"Allowing open host access to {subsystem}: Successful" in caplog.text else: - assert f"Added host {host} access to {subsystem}: True" in caplog.text + assert f"Adding host {host} to {subsystem}: Successful" in caplog.text @pytest.mark.parametrize("listener", listener_list) def test_create_listener(self, caplog, listener, gateway): caplog.clear() - cli(["create_listener", "-n", subsystem] + listener) + cli(["listener", "add", "--subsystem", subsystem] + listener) assert "enable_ha: False" in caplog.text assert "ipv4" in caplog.text.lower() - assert f"Created {subsystem} listener at {listener[3]}:{listener[5]}: True" in caplog.text + assert f"Adding {subsystem} listener at {listener[3]}:{listener[5]}: Successful" in caplog.text + assert f"auto HA state: AUTO_HA_UNSET" in caplog.text @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6) def test_create_listener_ipv6(self, caplog, listener_ipv6, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "create_listener", "-n", subsystem] + listener_ipv6) + cli(["--server-address", server_addr_ipv6, "listener", "add", "--subsystem", subsystem] + listener_ipv6) assert "enable_ha: False" in caplog.text assert "ipv6" in caplog.text.lower() - assert f"Created {subsystem} listener at [{listener_ipv6[3]}]:{listener_ipv6[5]}: True" in caplog.text + assert f"Adding {subsystem} listener at [{listener_ipv6[3]}]:{listener_ipv6[5]}: Successful" in caplog.text + assert f"auto HA state: AUTO_HA_UNSET" in caplog.text @pytest.mark.parametrize("listener", listener_list_no_port) def test_create_listener_no_port(self, caplog, listener, gateway): caplog.clear() - cli(["create_listener", "-n", subsystem] + listener) + cli(["listener", "add", "--subsystem", subsystem] + listener) assert "enable_ha: False" in caplog.text assert "ipv4" in caplog.text.lower() - assert f"Created {subsystem} listener at {listener[3]}:4420: True" in caplog.text + assert f"Adding {subsystem} listener at {listener[3]}:4420: Successful" in caplog.text + assert f"auto HA state: AUTO_HA_UNSET" in caplog.text + + @pytest.mark.parametrize("listener", listener_list_negative_port) + def test_create_listener_negative_port(self, caplog, listener, gateway): + caplog.clear() + rc = 0 + try: + cli(["listener", "add", "--subsystem", subsystem] + listener) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert "error: trsvcid value must be positive" in caplog.text + assert rc == 2 + + def test_create_listener_wrong_ha_state(self, caplog, gateway): + gw, stub = gateway + caplog.clear() + listener_add_req = pb2.create_listener_req(nqn=subsystem, gateway_name=gateway_name, trtype="TCP", + adrfam="ipv4", traddr=addr, trsvcid=5021, auto_ha_state="AUTO_HA_ON") + ret = stub.create_listener(listener_add_req) + assert "ipv4" in caplog.text.lower() + assert f"auto HA state: AUTO_HA_ON" in caplog.text + assert f"auto_ha_state is set to AUTO_HA_ON but we are not in an update()" in caplog.text @pytest.mark.parametrize("listener", listener_list_fc_trtype) def test_create_listener_fc_trtype(self, caplog, listener, gateway): caplog.clear() - rc = 0 - with pytest.raises(Exception) as ex: - try: - cli(["create_listener", "-n", subsystem] + listener) - except SystemExit as sysex: - rc = sysex - pass - assert "create_listener failed" in str(ex.value) - assert rc != 0 - assert "create_listener failed" in caplog.text - assert "Invalid parameters" in caplog.text + cli(["listener", "add", "--subsystem", subsystem] + listener) + assert f"Failure adding {subsystem} listener at {listener[3]}:{listener[5]}: Invalid parameters" in caplog.text assert '"trtype": "FC"' in caplog.text @pytest.mark.parametrize("listener", listener_list_invalid_trtype) def test_create_listener_invalid_trtype(self, caplog, listener, gateway): + caplog.clear() + rc = 0 + try: + cli(["listener", "add", "--subsystem", subsystem] + listener) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert "error: argument --trtype/-t: invalid choice: 'JUNK'" in caplog.text + assert rc == 2 + + def test_create_listener_invalid_trtype_no_cli(self, caplog, gateway): caplog.clear() with pytest.raises(Exception) as ex: - try: - cli(["create_listener", "-n", subsystem] + listener) - except SystemExit as sysex: - pass - assert "unknown enum label" in str(ex.value) - assert "unknown enum label" in caplog.text - assert f"Created {subsystem} listener at {listener[3]}:{listener[5]}: False" in caplog.text + listener_add_req = pb2.create_listener_req(nqn=subsystem, gateway_name=gateway_name, trtype="JUNK", + adrfam="ipv4", traddr=addr, trsvcid=5031, auto_ha_state="AUTO_HA_UNSET") + assert f'ValueError: unknown enum label "JUNK"' in str(ex.value) @pytest.mark.parametrize("listener", listener_list_invalid_adrfam) def test_create_listener_invalid_adrfam(self, caplog, listener, gateway): caplog.clear() - with pytest.raises(Exception) as ex: - try: - cli(["create_listener", "-n", subsystem] + listener) - except SystemExit as sysex: - pass - assert "unknown enum label" in str(ex.value) - assert "unknown enum label" in caplog.text - assert f"Created {subsystem} listener at {listener[3]}:{listener[5]}: False" in caplog.text + rc = 0 + try: + cli(["listener", "add", "--subsystem", subsystem] + listener) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert "error: argument --adrfam/-f: invalid choice: 'JUNK'" in caplog.text + assert rc == 2 @pytest.mark.parametrize("listener", listener_list_ib_adrfam) def test_create_listener_ib_adrfam(self, caplog, listener, gateway): caplog.clear() - rc = 0 - with pytest.raises(Exception) as ex: - try: - cli(["create_listener", "-n", subsystem] + listener) - except SystemExit as sysex: - rc = sysex - pass - assert "create_listener failed" in str(ex.value) - assert rc != 0 - assert "create_listener failed" in caplog.text + cli(["listener", "add", "--subsystem", subsystem] + listener) + assert f"Failure adding {subsystem} listener at {listener[3]}:{listener[5]}" in caplog.text assert "Invalid parameters" in caplog.text assert '"adrfam": "ib"' in caplog.text @pytest.mark.parametrize("listener", listener_list_discovery) def test_create_listener_on_discovery(self, caplog, listener, gateway): caplog.clear() - rc = 0 - with pytest.raises(Exception) as ex: - try: - cli(["create_listener"] + listener) - except SystemExit as sysex: - rc = sysex - pass - assert "Can't create a listener for a discovery subsystem" in str(ex.value) - assert rc != 0 + cli(["listener", "add"] + listener) assert "Can't create a listener for a discovery subsystem" in caplog.text class TestDelete: @pytest.mark.parametrize("host", host_list) def test_remove_host(self, caplog, host, gateway): caplog.clear() - cli(["remove_host", "-n", subsystem, "-t", host]) + cli(["host", "del", "--subsystem", subsystem, "--host", host]) if host == "*": - assert f"Disabled open host access to {subsystem}: True" in caplog.text + assert f"Disabling open host access to {subsystem}: Successful" in caplog.text else: - assert f"Removed host {host} access from {subsystem}: True" in caplog.text + assert f"Removing host {host} access from {subsystem}: Successful" in caplog.text @pytest.mark.parametrize("listener", listener_list) def test_delete_listener(self, caplog, listener, gateway): caplog.clear() - cli(["delete_listener", "-n", subsystem] + listener) - assert f"Deleted {listener[3]}:{listener[5]} from {subsystem}: True" in caplog.text + cli(["listener", "del", "--subsystem", subsystem] + listener) + assert f"Deleting listener {listener[3]}:{listener[5]} from {subsystem}: Successful" in caplog.text @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6) def test_delete_listener_ipv6(self, caplog, listener_ipv6, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "delete_listener", "-n", subsystem] + listener_ipv6) - assert f"Deleted [{listener_ipv6[3]}]:{listener_ipv6[5]} from {subsystem}: True" in caplog.text + cli(["--server-address", server_addr_ipv6, "listener", "del", "--subsystem", subsystem] + listener_ipv6) + assert f"Deleting listener [{listener_ipv6[3]}]:{listener_ipv6[5]} from {subsystem}: Successful" in caplog.text @pytest.mark.parametrize("listener", listener_list_no_port) def test_delete_listener_no_port(self, caplog, listener, gateway): caplog.clear() - cli(["delete_listener", "-n", subsystem] + listener) - assert f"Deleted {listener[3]}:4420 from {subsystem}: True" in caplog.text + cli(["listener", "del", "--subsystem", subsystem] + listener) + assert f"Deleting listener {listener[3]}:4420 from {subsystem}: Successful" in caplog.text def test_remove_namespace(self, caplog, gateway): + gw, stub = gateway caplog.clear() - cli(["remove_namespace", "-n", subsystem, "-i", nsid]) - assert f"Removed namespace {nsid} from {subsystem}: True" in caplog.text - cli(["remove_namespace", "-n", subsystem, "-i", nsid_ipv6]) - assert f"Removed namespace {nsid_ipv6} from {subsystem}: True" in caplog.text + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", nsid]) + bdev_name = "" + bdev_index = caplog.text.find('"bdev_name": "') + if bdev_index >= 0: + bdev_name = caplog.text[bdev_index + len('"bdev_name": "') :] + bdev_index = bdev_name.find('"') + if bdev_index >= 0: + bdev_name = bdev_name[0 : bdev_index] + else: + bdev_name = "" + assert bdev_name + caplog.clear() + cli(["namespace", "del", "--subsystem", subsystem, "--nsid", nsid]) + assert f"Deleting namespace {nsid} from {subsystem}: Successful" in caplog.text + bdev_list = rpc_bdev.bdev_get_bdevs(gw.spdk_rpc_client) + for b in bdev_list: + b_name = b["name"] + assert bdev_name != b_name + cli(["namespace", "del", "--subsystem", subsystem, "--nsid", nsid_ipv6]) + assert f"Deleting namespace {nsid_ipv6} from {subsystem}: Successful" in caplog.text - def test_delete_bdev(self, caplog, gateway): + def test_delete_subsystem(self, caplog, gateway): caplog.clear() - cli(["delete_bdev", "-b", bdev, "-f"]) - assert f"Deleted bdev {bdev}: True" in caplog.text - assert "Will remove namespace" not in caplog.text + cli(["subsystem", "del", "--subsystem", subsystem]) + assert f"Failure deleting subsystem {subsystem}: Namespace 2 is still using the subsystem" caplog.clear() - # Should fail as there is a namespace using the bdev - rc = 0 - with pytest.raises(Exception) as ex: - try: - cli(["delete_bdev", "-b", bdev1]) - except SystemExit as sysex: - # should fail with non-zero return code - rc = sysex - assert "Device or resource busy" in str(ex.value) - assert rc != 0 - assert "Device or resource busy" in caplog.text - assert f"Namespace 2 from {subsystem} is still using bdev {bdev1}" in caplog.text - caplog.clear() - cli(["delete_bdev", "-b", bdev1, "--force"]) - assert f"Deleted bdev {bdev1}: True" in caplog.text - assert f"Removed namespace 2 from {subsystem}" in caplog.text + cli(["subsystem", "del", "--subsystem", subsystem, "--force"]) + assert f"Deleting subsystem {subsystem}: Successful" in caplog.text caplog.clear() - cli(["delete_bdev", "-b", bdev_ipv6, "-f"]) - assert f"Deleted bdev {bdev_ipv6}: True" in caplog.text - assert "Will remove namespace" not in caplog.text + cli(["subsystem", "del", "--subsystem", subsystem2]) + assert f"Deleting subsystem {subsystem2}: Successful" in caplog.text caplog.clear() - cli(["delete_bdev", "-b", bdev1_ipv6, "--force"]) - assert f"Deleted bdev {bdev1_ipv6}: True" in caplog.text - assert f"Removed namespace 4 from {subsystem}" in caplog.text + cli(["subsystem", "list"]) + assert "No subsystems" in caplog.text - def test_delete_subsystem(self, caplog, gateway): - caplog.clear() - cli(["delete_subsystem", "-n", subsystem]) - assert f"Deleted subsystem {subsystem}: True" in caplog.text + def test_delete_subsystem_with_discovery_nqn(self, caplog, gateway): caplog.clear() - cli(["delete_subsystem", "-n", subsystem2]) - assert f"Deleted subsystem {subsystem2}: True" in caplog.text + rc = 0 + try: + cli(["subsystem", "del", "--subsystem", discovery_nqn]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert "Can't delete a discovery subsystem" in caplog.text + assert rc == 2 class TestCreateWithAna: - def test_create_bdev_ana(self, caplog, gateway): - caplog.clear() - cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev]) - assert f"Created bdev {bdev}: True" in caplog.text - - def test_create_bdev_ana_ipv6(self, caplog, gateway): - caplog.clear() - cli(["--server-address", server_addr_ipv6, "create_bdev", "-i", image, "-p", pool, "-b", bdev_ipv6]) - assert f"Created bdev {bdev_ipv6}: True" in caplog.text - def test_create_subsystem_ana(self, caplog, gateway): caplog.clear() + cli(["subsystem", "list"]) + assert "No subsystems" in caplog.text rc = 0 - with pytest.raises(Exception) as ex: - try: - cli(["create_subsystem", "-n", subsystem, "-t"]) - except SystemExit as sysex: - # should fail with non-zero return code - rc = sysex - assert "HA enabled but ANA-reporting is disabled" in str(ex.value) + try: + cli(["subsystem", "add", "--subsystem", subsystem, "--enable-ha"]) + except SystemExit as sysex: + # should fail with non-zero return code + rc = int(str(sysex)) + assert "ANA reporting must be enabled when HA is active" in caplog.text assert rc != 0 - assert "HA enabled but ANA-reporting is disabled" in caplog.text caplog.clear() - cli(["create_subsystem", "-n", subsystem, "-a", "-t"]) - assert f"Created subsystem {subsystem}: True" in caplog.text + cli(["subsystem", "list"]) + assert "No subsystems" in caplog.text + caplog.clear() + cli(["subsystem", "add", "--subsystem", subsystem, "--ana-reporting", "--enable-ha"]) + assert f"Adding subsystem {subsystem}: Successful" in caplog.text assert "ana reporting: True" in caplog.text caplog.clear() - cli(["get_subsystems"]) + cli(["subsystem", "list"]) assert serial not in caplog.text + assert subsystem in caplog.text def test_add_namespace_ana(self, caplog, gateway): caplog.clear() - cli(["add_namespace", "-n", subsystem, "-b", bdev, "-a", anagrpid]) - assert f"Added namespace 1 to {subsystem}, ANA group id {anagrpid}" in caplog.text + cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--load-balancing-group", anagrpid]) + assert f"Adding namespace {nsid} to {subsystem}, load balancing group {anagrpid}: Successful" in caplog.text + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", nsid]) + assert f'"load_balancing_group": {anagrpid}' in caplog.text + + def test_change_namespace_lb_group(self, caplog, gateway): + caplog.clear() + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", nsid]) + assert f'"load_balancing_group": {anagrpid}' in caplog.text + caplog.clear() + cli(["namespace", "change_load_balancing_group", "--subsystem", subsystem, "--nsid", nsid, "--load-balancing-group", anagrpid2]) + assert f"Changing load balancing group of namespace {nsid} in {subsystem} to {anagrpid2}: Successful" in caplog.text caplog.clear() - cli(["get_subsystems"]) - assert "failed" not in caplog.text + cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", nsid]) + assert f'"load_balancing_group": {anagrpid2}' in caplog.text + assert f'"load_balancing_group": {anagrpid}' not in caplog.text @pytest.mark.parametrize("listener", listener_list) def test_create_listener_ana(self, caplog, listener, gateway): caplog.clear() - cli(["create_listener", "-n", subsystem] + listener) + cli(["listener", "add", "--subsystem", subsystem] + listener) assert "enable_ha: True" in caplog.text assert "ipv4" in caplog.text.lower() - assert f"Created {subsystem} listener at {listener[3]}:{listener[5]}: True" in caplog.text - + assert f"Adding {subsystem} listener at {listener[3]}:{listener[5]}: Successful" in caplog.text + assert f"auto HA state: AUTO_HA_UNSET" in caplog.text class TestDeleteAna: @pytest.mark.parametrize("listener", listener_list) def test_delete_listener_ana(self, caplog, listener, gateway): caplog.clear() - cli(["delete_listener", "-n", subsystem] + listener) - assert f"Deleted {listener[3]}:{listener[5]} from {subsystem}: True" in caplog.text + cli(["listener", "del", "--subsystem", subsystem] + listener) + assert f"Deleting listener {listener[3]}:{listener[5]} from {subsystem}: Successful" in caplog.text def test_remove_namespace_ana(self, caplog, gateway): caplog.clear() - cli(["remove_namespace", "-n", subsystem, "-i", nsid]) - assert f"Removed namespace 1 from {subsystem}: True" in caplog.text - - def test_delete_bdev_ana(self, caplog, gateway): - caplog.clear() - cli(["delete_bdev", "-b", bdev, "-f"]) - assert f"Deleted bdev {bdev}: True" in caplog.text - assert "Will remove namespace" not in caplog.text - caplog.clear() - cli(["delete_bdev", "-b", bdev_ipv6, "-f"]) - assert f"Deleted bdev {bdev_ipv6}: True" in caplog.text - assert "Will remove namespace" not in caplog.text + cli(["namespace", "del", "--subsystem", subsystem, "--nsid", nsid]) + assert f"Deleting namespace {nsid} from {subsystem}: Successful" in caplog.text def test_delete_subsystem_ana(self, caplog, gateway): caplog.clear() - cli(["delete_subsystem", "-n", subsystem]) - assert f"Deleted subsystem {subsystem}: True" in caplog.text + cli(["subsystem", "del", "--subsystem", subsystem]) + assert f"Deleting subsystem {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["subsystem", "list"]) + assert "No subsystems" in caplog.text class TestSPDKLOg: def test_log_flags(self, caplog, gateway): caplog.clear() - cli(["get_spdk_nvmf_log_flags_and_level"]) - assert '"nvmf": false' in caplog.text - assert '"nvmf_tcp": false' in caplog.text - assert '"log_level": "NOTICE"' in caplog.text - assert '"log_print_level": "INFO"' in caplog.text + cli(["log_level", "get"]) + assert 'SPDK nvmf log flag "nvmf" is disabled' in caplog.text + assert 'SPDK nvmf log flag "nvmf_tcp" is disabled' in caplog.text + assert 'SPDK log level is NOTICE' in caplog.text + assert 'SPDK log print level is INFO' in caplog.text caplog.clear() - cli(["set_spdk_nvmf_logs"]) - assert "Set SPDK nvmf logs: True" in caplog.text + cli(["log_level", "set"]) + assert "Set SPDK log levels and nvmf log flags: Successful" in caplog.text caplog.clear() - cli(["get_spdk_nvmf_log_flags_and_level"]) - assert '"nvmf": true' in caplog.text - assert '"nvmf_tcp": true' in caplog.text - assert '"log_level": "NOTICE"' in caplog.text - assert '"log_print_level": "INFO"' in caplog.text + cli(["log_level", "get"]) + assert 'SPDK nvmf log flag "nvmf" is enabled' in caplog.text + assert 'SPDK nvmf log flag "nvmf_tcp" is enabled' in caplog.text + assert 'SPDK log level is NOTICE' in caplog.text + assert 'SPDK log print level is INFO' in caplog.text caplog.clear() - cli(["set_spdk_nvmf_logs", "-l", "DebuG"]) - assert "Set SPDK nvmf logs: True" in caplog.text + cli(["log_level", "set", "--level", "DEBUG"]) + assert "Set SPDK log levels and nvmf log flags: Successful" in caplog.text caplog.clear() - cli(["get_spdk_nvmf_log_flags_and_level"]) - assert '"nvmf": true' in caplog.text - assert '"nvmf_tcp": true' in caplog.text - assert '"log_level": "DEBUG"' in caplog.text - assert '"log_print_level": "INFO"' in caplog.text + cli(["log_level", "get"]) + assert 'SPDK nvmf log flag "nvmf" is enabled' in caplog.text + assert 'SPDK nvmf log flag "nvmf_tcp" is enabled' in caplog.text + assert 'SPDK log level is DEBUG' in caplog.text + assert 'SPDK log print level is INFO' in caplog.text caplog.clear() - cli(["set_spdk_nvmf_logs", "-p", "eRRor"]) - assert "Set SPDK nvmf logs: True" in caplog.text + cli(["log_level", "set", "--print", "error"]) + assert "Set SPDK log levels and nvmf log flags: Successful" in caplog.text caplog.clear() - cli(["get_spdk_nvmf_log_flags_and_level"]) - assert '"nvmf": true' in caplog.text - assert '"nvmf_tcp": true' in caplog.text - assert '"log_level": "DEBUG"' in caplog.text - assert '"log_print_level": "ERROR"' in caplog.text + cli(["log_level", "get"]) + assert 'SPDK nvmf log flag "nvmf" is enabled' in caplog.text + assert 'SPDK nvmf log flag "nvmf_tcp" is enabled' in caplog.text + assert 'SPDK log level is DEBUG' in caplog.text + assert 'SPDK log print level is ERROR' in caplog.text caplog.clear() - cli(["disable_spdk_nvmf_logs"]) - assert "Disable SPDK nvmf logs: True" in caplog.text + cli(["log_level", "disable"]) + assert "Disable SPDK nvmf log flags: Successful" in caplog.text caplog.clear() - cli(["get_spdk_nvmf_log_flags_and_level"]) - assert '"nvmf": false' in caplog.text - assert '"nvmf_tcp": false' in caplog.text - assert '"log_level": "NOTICE"' in caplog.text - assert '"log_print_level": "INFO"' in caplog.text + cli(["log_level", "get"]) + assert 'SPDK nvmf log flag "nvmf" is disabled' in caplog.text + assert 'SPDK nvmf log flag "nvmf_tcp" is disabled' in caplog.text + assert 'SPDK log level is NOTICE' in caplog.text + assert 'SPDK log print level is INFO' in caplog.text caplog.clear() - with pytest.raises(Exception) as ex: - try: - cli(["set_spdk_nvmf_logs", "-l", "JUNK"]) - except SystemExit as sysex: - pass - assert "Set SPDK nvmf logs: False" in str(ex.value) - assert "Set SPDK nvmf logs: False" in caplog.text + rc = 0 + try: + cli(["log_level", "set", "-l", "JUNK"]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert "error: argument --level/-l: invalid choice: 'JUNK'" in caplog.text + assert rc == 2 diff --git a/tests/test_grpc.py b/tests/test_grpc.py index 74284ee36..c328163e7 100644 --- a/tests/test_grpc.py +++ b/tests/test_grpc.py @@ -9,24 +9,23 @@ logger = logging.getLogger(__name__) image = "mytestdevimage" pool = "rbd" -bdev_prefix = "Ceph0" subsystem_prefix = "nqn.2016-06.io.spdk:cnode" created_resource_count = 500 -get_subsys_count = 100 +subsys_list_count = 100 def create_resource_by_index(i): - bdev = f"{bdev_prefix}_{i}" - cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev]) subsystem = f"{subsystem_prefix}{i}" - cli(["create_subsystem", "-n", subsystem, "-a", "-t" ]) - cli(["add_namespace", "-n", subsystem, "-b", bdev]) + cli(["subsystem", "add", "--subsystem", subsystem, "--ana-reporting", "--enable-ha" ]) + cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image]) def check_resource_by_index(i, caplog): - bdev = f"{bdev_prefix}_{i}" - # notice that this also verifies the namespace as the bdev name is in the namespaces section - assert f"{bdev}" in caplog.text subsystem = f"{subsystem_prefix}{i}" + caplog.clear() + cli(["subsystem", "list"]) assert f"{subsystem}" in caplog.text + caplog.clear() + cli(["namespace", "list", "--subsystem", subsystem, "--nsid", "1"]) + assert f"No namespace" not in caplog.text # We want to fail in case we got an exception about invalid data in pb2 functions but this is just a warning # for pytest. In order for the test to fail in such a case we need to ask pytest to regard this as an error @@ -40,7 +39,8 @@ def test_create_get_subsys(caplog, config): assert "failed" not in caplog.text.lower() # add a listener - cli(["create_listener", "-n", f"{subsystem_prefix}0", "-g", gateway.name, "-a", "127.0.0.1", "-s", "5001"]) + cli(["listener", "add", "--subsystem", f"{subsystem_prefix}0", "--gateway-name", + gateway.name, "--traddr", "127.0.0.1", "--trsvcid", "5001"]) assert f"auto HA state: AUTO_HA_UNSET" in caplog.text caplog.clear() @@ -49,16 +49,16 @@ def test_create_get_subsys(caplog, config): with GatewayServer(config) as gateway: gateway.serve() - for i in range(get_subsys_count): - cli(["get_subsystems"]) + for i in range(subsys_list_count): + cli(["subsystem", "list"]) assert "Exception" not in caplog.text time.sleep(0.1) time.sleep(20) # Make sure update() is over assert f"auto HA state: AUTO_HA_ON" in caplog.text caplog.clear() - cli(["get_subsystems"]) + cli(["subsystem", "list"]) assert "Exception" not in caplog.text - assert "get_subsystems: []" not in caplog.text + assert "No subsystems" not in caplog.text for i in range(created_resource_count): check_resource_by_index(i, caplog) diff --git a/tests/test_multi_gateway.py b/tests/test_multi_gateway.py index 607878b1c..f3295cc7d 100644 --- a/tests/test_multi_gateway.py +++ b/tests/test_multi_gateway.py @@ -67,7 +67,6 @@ def test_multi_gateway_coordination(config, image, conn): periodic polling. """ stubA, stubB = conn - bdev = "Ceph0" nqn = "nqn.2016-06.io.spdk:cnode1" serial = "SPDK00000000000001" nsid = 10 @@ -76,43 +75,62 @@ def test_multi_gateway_coordination(config, image, conn): pool = config.get("ceph", "pool") # Send requests to create a subsystem with one namespace to GatewayA - bdev_req = pb2.create_bdev_req(bdev_name=bdev, - rbd_pool_name=pool, - rbd_image_name=image, - block_size=4096) subsystem_req = pb2.create_subsystem_req(subsystem_nqn=nqn, serial_number=serial) - namespace_req = pb2.add_namespace_req(subsystem_nqn=nqn, - bdev_name=bdev, + namespace_req = pb2.namespace_add_req(subsystem_nqn=nqn, + rbd_pool_name=pool, + rbd_image_name=image, + block_size=4096, nsid=nsid) - get_subsystems_req = pb2.get_subsystems_req() - ret_bdev = stubA.create_bdev(bdev_req) + list_subsystems_req = pb2.list_subsystems_req() + list_namespaces_req = pb2.list_namespaces_req(subsystem=nqn) ret_subsystem = stubA.create_subsystem(subsystem_req) - ret_namespace = stubA.add_namespace(namespace_req) - assert ret_bdev.status is True - assert ret_subsystem.status is True - assert ret_namespace.status is True + ret_namespace = stubA.namespace_add(namespace_req) + assert ret_subsystem.status == 0 + assert ret_namespace.status == 0 + + nsListA = json.loads(json_format.MessageToJson( + stubA.list_namespaces(list_namespaces_req), + preserving_proto_field_name=True, including_default_value_fields=True))['namespaces'] + assert len(nsListA) == 1 + assert nsListA[0]["nsid"] == nsid + bdev_name = nsListA[0]["bdev_name"] # Watch/Notify if update_notify: time.sleep(1) listB = json.loads(json_format.MessageToJson( - stubB.get_subsystems(get_subsystems_req), + stubB.list_subsystems(list_subsystems_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] assert len(listB) == num_subsystems assert listB[num_subsystems-1]["nqn"] == nqn assert listB[num_subsystems-1]["serial_number"] == serial - assert listB[num_subsystems-1]["namespaces"][0]["nsid"] == nsid - assert listB[num_subsystems-1]["namespaces"][0]["bdev_name"] == bdev + assert listB[num_subsystems-1]["namespace_count"] == 1 + + nsListB = json.loads(json_format.MessageToJson( + stubB.list_namespaces(list_namespaces_req), + preserving_proto_field_name=True, including_default_value_fields=True))['namespaces'] + assert len(nsListB) == 1 + assert nsListB[0]["nsid"] == nsid + assert nsListB[0]["bdev_name"] == bdev_name + assert nsListB[0]["rbd_image_name"] == image + assert nsListB[0]["rbd_pool_name"] == pool # Periodic update time.sleep(update_interval_sec + 1) listB = json.loads(json_format.MessageToJson( - stubB.get_subsystems(get_subsystems_req), + stubB.list_subsystems(list_subsystems_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] assert len(listB) == num_subsystems assert listB[num_subsystems-1]["nqn"] == nqn assert listB[num_subsystems-1]["serial_number"] == serial - assert listB[num_subsystems-1]["namespaces"][0]["nsid"] == nsid - assert listB[num_subsystems-1]["namespaces"][0]["bdev_name"] == bdev + assert listB[num_subsystems-1]["namespace_count"] == 1 + nsListB = json.loads(json_format.MessageToJson( + stubB.list_namespaces(list_namespaces_req), + preserving_proto_field_name=True, including_default_value_fields=True))['namespaces'] + assert len(nsListB) == 1 + assert nsListB[0]["nsid"] == nsid + assert nsListB[0]["bdev_name"] == bdev_name + assert nsListB[0]["rbd_image_name"] == image + assert nsListB[0]["rbd_pool_name"] == pool diff --git a/tests/test_omap_lock.py b/tests/test_omap_lock.py index e552ca1b7..f6ea2cbf4 100644 --- a/tests/test_omap_lock.py +++ b/tests/test_omap_lock.py @@ -11,7 +11,6 @@ image = "mytestdevimage" pool = "rbd" -bdev_prefix = "Ceph_" subsystem_prefix = "nqn.2016-06.io.spdk:cnode" host_nqn_prefix = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b" created_resource_count = 200 @@ -134,129 +133,103 @@ def build_host_nqn(i): return hostnqn def create_resource_by_index(stub, i, caplog): - bdev = f"{bdev_prefix}{i}" - bdev_req = pb2.create_bdev_req(bdev_name=bdev, - rbd_pool_name=pool, - rbd_image_name=image, - block_size=4096) - ret_bdev = stub.create_bdev(bdev_req) - assert ret_bdev.status - if caplog != None: - assert f"create_bdev: {bdev}" in caplog.text - assert "create_bdev failed" not in caplog.text subsystem = f"{subsystem_prefix}{i}" subsystem_req = pb2.create_subsystem_req(subsystem_nqn=subsystem) ret_subsystem = stub.create_subsystem(subsystem_req) - assert ret_subsystem.status + assert ret_subsystem.status == 0 if caplog != None: assert f"create_subsystem {subsystem}: True" in caplog.text - assert "create_subsystem failed" not in caplog.text - namespace_req = pb2.add_namespace_req(subsystem_nqn=subsystem, - bdev_name=bdev) - ret_namespace = stub.add_namespace(namespace_req) - assert ret_namespace.status + assert f"Failure creating subsystem {subsystem}" not in caplog.text + namespace_req = pb2.namespace_add_req(subsystem_nqn=subsystem, + rbd_pool_name=pool, rbd_image_name=image, block_size=4096) + ret_namespace = stub.namespace_add(namespace_req) + assert ret_namespace.status == 0 hostnqn = build_host_nqn(i) host_req = pb2.add_host_req(subsystem_nqn=subsystem, host_nqn=hostnqn) ret_host = stub.add_host(host_req) - assert ret_host.status + assert ret_host.status == 0 host_req = pb2.add_host_req(subsystem_nqn=subsystem, host_nqn="*") ret_host = stub.add_host(host_req) - assert ret_host.status + assert ret_host.status == 0 if caplog != None: assert f"add_host {hostnqn}: True" in caplog.text assert "add_host *: True" in caplog.text - assert "add_host failed" not in caplog.text + assert f"Failure allowing open host access to {subsystem}" not in caplog.text + assert f"Failure adding host {hostnqn} to {subsystem}" not in caplog.text -def check_resource_by_index(i, resource_list): - # notice that this also verifies the namespace as the bdev name is in the namespaces section - bdev = f"{bdev_prefix}{i}" +def check_resource_by_index(i, subsys_list, hosts_info): subsystem = f"{subsystem_prefix}{i}" hostnqn = build_host_nqn(i) - found_bdev = False found_host = False - for res in resource_list: + for subsys in subsys_list: try: - if res["nqn"] != subsystem: + if subsys["nqn"] != subsystem: continue - assert res["allow_any_host"] - for host in res["hosts"]: + assert subsys["namespace_count"] == 1 + assert hosts_info["allow_any_host"] + for host in hosts_info["hosts"]: if host["nqn"] == hostnqn: found_host = True - for ns in res["namespaces"]: - if ns["bdev_name"] == bdev: - found_bdev = True - break break except Exception: pass - assert found_bdev and found_host + assert found_host def test_multi_gateway_omap_reread(config, conn_omap_reread, caplog): """Tests reading out of date OMAP file """ stubA, stubB, gatewayA, gatewayB = conn_omap_reread - bdev = bdev_prefix + "X0" - bdev2 = bdev_prefix + "X1" - bdev3 = bdev_prefix + "X2" nqn = subsystem_prefix + "X1" serial = "SPDK00000000000001" nsid = 10 num_subsystems = 2 # Send requests to create a subsystem with one namespace to GatewayA - bdev_req = pb2.create_bdev_req(bdev_name=bdev, - rbd_pool_name=pool, - rbd_image_name=image, - block_size=4096) - subsystem_req = pb2.create_subsystem_req(subsystem_nqn=nqn, - serial_number=serial) - namespace_req = pb2.add_namespace_req(subsystem_nqn=nqn, - bdev_name=bdev, - nsid=nsid) - get_subsystems_req = pb2.get_subsystems_req() - ret_bdev = stubA.create_bdev(bdev_req) + subsystem_req = pb2.create_subsystem_req(subsystem_nqn=nqn, serial_number=serial) + namespace_req = pb2.namespace_add_req(subsystem_nqn=nqn, nsid=nsid, + rbd_pool_name=pool, rbd_image_name=image, block_size=4096) + + subsystem_list_req = pb2.list_subsystems_req() ret_subsystem = stubA.create_subsystem(subsystem_req) - ret_namespace = stubA.add_namespace(namespace_req) - assert ret_bdev.status is True - assert ret_subsystem.status is True - assert ret_namespace.status is True + assert ret_subsystem.status == 0 + ret_namespace = stubA.namespace_add(namespace_req) + assert ret_namespace.status == 0 # Until we create some resource on GW-B it shouldn't still have the resrouces created on GW-A, only the discovery subsystem listB = json.loads(json_format.MessageToJson( - stubB.get_subsystems(get_subsystems_req), + stubB.list_subsystems(subsystem_list_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] assert len(listB) == 1 listA = json.loads(json_format.MessageToJson( - stubA.get_subsystems(get_subsystems_req), + stubA.list_subsystems(subsystem_list_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] assert len(listA) == num_subsystems - bdev2_req = pb2.create_bdev_req(bdev_name=bdev2, + ns2_req = pb2.namespace_add_req(subsystem_nqn=nqn, rbd_pool_name=pool, rbd_image_name=image, block_size=4096) - ret_bdev2 = stubB.create_bdev(bdev2_req) - assert ret_bdev2.status is True + ret_ns2 = stubB.namespace_add(ns2_req) + assert ret_ns2.status == 0 assert "The file is not current, will reload it and try again" in caplog.text # Make sure that after reading the OMAP file GW-B has the subsystem and namespace created on GW-A listB = json.loads(json_format.MessageToJson( - stubB.get_subsystems(get_subsystems_req), + stubB.list_subsystems(subsystem_list_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] assert len(listB) == num_subsystems assert listB[num_subsystems-1]["nqn"] == nqn assert listB[num_subsystems-1]["serial_number"] == serial - assert listB[num_subsystems-1]["namespaces"][0]["nsid"] == nsid - assert listB[num_subsystems-1]["namespaces"][0]["bdev_name"] == bdev + assert listB[num_subsystems-1]["namespace_count"] == num_subsystems # We created one namespace on each subsystem caplog.clear() - bdev3_req = pb2.create_bdev_req(bdev_name=bdev3, + ns3_req = pb2.namespace_add_req(subsystem_nqn=nqn, rbd_pool_name=pool, rbd_image_name=image, block_size=4096) - ret_bdev3 = stubB.create_bdev(bdev3_req) - assert ret_bdev3.status is True + ret_ns3 = stubB.namespace_add(ns3_req) + assert ret_ns3.status == 0 assert "The file is not current, will reload it and try again" not in caplog.text bdevsA = rpc_bdev.bdev_get_bdevs(gatewayA.spdk_rpc_client) @@ -265,10 +238,7 @@ def test_multi_gateway_omap_reread(config, conn_omap_reread, caplog): # GW-A should only have the bdev created on it as we didn't update it after creating the bdev on GW-B assert len(bdevsA) == 1 assert len(bdevsB) == 3 - assert bdevsA[0]["name"] == bdev - assert bdevsB[0]["name"] == bdev - assert bdevsB[1]["name"] == bdev2 - assert bdevsB[2]["name"] == bdev3 + assert bdevsA[0]["name"] == bdevsB[0]["name"] def test_trying_to_lock_twice(config, image, conn_lock_twice, caplog): """Tests an attempt to lock the OMAP file from two gateways at the same time @@ -300,13 +270,21 @@ def test_multi_gateway_concurrent_changes(config, image, conn_concurrent, caplog # Let the update some time to bring both gateways to the same page time.sleep(15) caplog.clear() - get_subsystems_req = pb2.get_subsystems_req() - listA = json.loads(json_format.MessageToJson( - stubA.get_subsystems(get_subsystems_req), + subsystem_list_req = pb2.list_subsystems_req() + subListA = json.loads(json_format.MessageToJson( + stubA.list_subsystems(subsystem_list_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] - listB = json.loads(json_format.MessageToJson( - stubB.get_subsystems(get_subsystems_req), + subListB = json.loads(json_format.MessageToJson( + stubB.list_subsystems(subsystem_list_req), preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] for i in range(created_resource_count): - check_resource_by_index(i, listA) - check_resource_by_index(i, listB) + subsystem = f"{subsystem_prefix}{i}" + host_list_req = pb2.list_hosts_req(subsystem=subsystem) + hostListA = json.loads(json_format.MessageToJson( + stubA.list_hosts(host_list_req), + preserving_proto_field_name=True, including_default_value_fields=True)) + hostListB = json.loads(json_format.MessageToJson( + stubB.list_hosts(host_list_req), + preserving_proto_field_name=True, including_default_value_fields=True)) + check_resource_by_index(i, subListA, hostListA) + check_resource_by_index(i, subListB, hostListB)