Skip to content

Commit

Permalink
New NVMeOf CLI
Browse files Browse the repository at this point in the history 
Fixes ceph#151

Signed-off-by: Gil Bregman <[email protected]>
  • Loading branch information
@gbregman
gbregman committed Dec 27, 2023
1 parent 90b60ba commit d447185
Show file tree
Hide file tree
Showing 18 changed files with 4,039 additions and 1,298 deletions.
37 changes: 26 additions & 11 deletions .github/workflows/build-container.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ jobs:
strategy:
fail-fast: false
matrix:
test: ["cli", "state", "multi_gateway", "server", "grpc", "omap_lock"]
test: ["cli", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap"]
runs-on: ubuntu-latest
env:
HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance
Expand Down Expand Up @@ -223,13 +223,29 @@ jobs:
run: |
make demo OPTS=-T NVMEOF_CONTAINER_NAME="ceph-nvmeof_nvmeof_1"
- name: Get subsystems
- name: List resources
run: |
# https://github.com/actions/toolkit/issues/766
shopt -s expand_aliases
eval $(make alias)
nvmeof-cli get_subsystems
nvmeof-cli-ipv6 get_subsystems
cephnvmf subsystem list
subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//')
for sub in $subs
do
cephnvmf namespace list --subsystem $sub
cephnvmf listener list --subsystem $sub
cephnvmf host list --subsystem $sub
done
- name: Verify no HA
run: |
shopt -s expand_aliases
eval $(make alias)
ha_status=$(cephnvmf --output stdio --format json subsystem list | grep "enable_ha" | grep "true" | tr -d '\n\r')
if [ -n "$ha_status" ]; then
echo "Found a subsystem with HA enabled, failing"
exit 1
fi
- name: Run bdevperf
run: |
Expand Down Expand Up @@ -390,21 +406,20 @@ jobs:
gw1=$(container_ip $GW1)
echo ℹ️ Using GW RPC $GW1 address $gw1 port $NVMEOF_GW_PORT
cli_gw $gw1 get_subsystems
cli_gw $gw1 create_bdev --pool $RBD_POOL --image $RBD_IMAGE_NAME --bdev $BDEV_NAME
cli_gw $gw1 create_subsystem --subnqn $NQN --serial $SERIAL
cli_gw $gw1 add_namespace --subnqn $NQN --bdev $BDEV_NAME
cli_gw $gw1 subsystem list
cli_gw $gw1 subsystem add --subsystem $NQN --serial $SERIAL
cli_gw $gw1 namespace add --subsystem $NQN --rbd-pool $RBD_POOL --rbd-image $RBD_IMAGE_NAME
for gw in $GW1 $GW2; do
ip=$(container_ip $gw)
name=$(container_id $gw) # default hostname - container id
echo ℹ️ Create listener address $ip gateway $name
cli_gw $ip create_listener --subnqn $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT
cli_gw $ip listener add --subsystem $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT
done
cli_gw $gw1 add_host --subnqn $NQN --host "*"
cli_gw $gw1 host add --subsystem $NQN --host "*"
for gw in $GW1 $GW2; do
ip=$(container_ip $gw)
echo ℹ️ Subsystems for name $gw ip $ip
cli_gw $ip get_subsystems
cli_gw $ip subsystem list
done
- name: Run bdevperf discovery
Expand Down
16 changes: 15 additions & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ FROM quay.io/ceph/spdk:${NVMEOF_SPDK_VERSION:-NULL} AS base-gateway
RUN \
--mount=type=cache,target=/var/cache/dnf \
--mount=type=cache,target=/var/lib/dnf \
dnf install -y python3-rados
dnf install -y python3-rados python3-pip
RUN \
--mount=type=cache,target=/root/.cache/pip \
pip install -U pip setuptools
ENTRYPOINT ["python3", "-m", "control"]
CMD ["-c", "/src/ceph-nvmeof.conf"]

Expand Down Expand Up @@ -99,6 +102,17 @@ LABEL io.ceph.component="$NVMEOF_NAME" \

ENV PYTHONPATH=$APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib

RUN \
--mount=type=cache,target=/var/cache/dnf \
--mount=type=cache,target=/var/lib/dnf \
dnf install -y python3-pip
RUN \
--mount=type=cache,target=/root/.cache/pip \
TABULATE_INSTALL=lib-only pip install -U --target $APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib tabulate
RUN \
--mount=type=cache,target=/root/.cache/pip \
pip install -U --target $APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib pyyaml

WORKDIR $APPDIR

#------------------------------------------------------------------------------
Expand Down
49 changes: 16 additions & 33 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,36 +92,25 @@ The following command executes all the steps required to set up the NVMe-oF envi

```bash
$ make demo
DOCKER_BUILDKIT=1 docker-compose exec ceph-vstart-cluster bash -c "rbd info demo_image || rbd create demo_image --size 10M"
docker-compose exec ceph bash -c "rbd -p rbd info demo_image || rbd -p rbd create demo_image --size 10M"
rbd: error opening image demo_image: (2) No such file or directory
DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_bdev --pool rbd --image demo_image --bdev demo_bdev
Creating nvmeof_ceph-nvmeof-cli_run ... done
INFO:__main__:Created bdev demo_bdev: True
DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_subsystem --subnqn nqn.2016-06.io.spdk:cnode1 --serial SPDK00000000000001
Creating nvmeof_ceph-nvmeof-cli_run ... done
INFO:__main__:Created subsystem nqn.2016-06.io.spdk:cnode1: True
DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 add_namespace --subnqn nqn.2016-06.io.spdk:cnode1 --bdev demo_bdev
Creating nvmeof_ceph-nvmeof-cli_run ... done
INFO:__main__:Added namespace 1 to nqn.2016-06.io.spdk:cnode1: True
DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_listener --subnqn nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420
Creating nvmeof_ceph-nvmeof-cli_run ... done
INFO:__main__:Created nqn.2016-06.io.spdk:cnode1 listener: True
DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 add_host --subnqn nqn.2016-06.io.spdk:cnode1 --host "*"
Creating nvmeof_ceph-nvmeof-cli_run ... done
INFO:__main__:Allowed open host access to nqn.2016-06.io.spdk:cnode1: True
docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 subsystem add --subsystem "nqn.2016-06.io.spdk:cnode1" --ana-reporting --enable-ha
Adding subsystem nqn.2016-06.io.spdk:cnode1: Successful
docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 namespace add --subsystem "nqn.2016-06.io.spdk:cnode1" --rbd-pool rbd --rbd-image demo_image
Adding namespace 1 to nqn.2016-06.io.spdk:cnode1, load balancing group 1: Successful
docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 listener add --subsystem "nqn.2016-06.io.spdk:cnode1" --gateway-name fbca1a3d3ed8 --traddr 192.168.13.3 --trsvcid 4420
Adding listener 192.168.13.3:4420 to nqn.2016-06.io.spdk:cnode1: Successful
docker-compose run --rm nvmeof-cli --server-address 2001:db8::3 --server-port 5500 listener add --subsystem "nqn.2016-06.io.spdk:cnode1" --gateway-name fbca1a3d3ed8 --traddr 2001:db8::3 --trsvcid 4420 --adrfam IPV6
Adding listener [2001:db8::3]:4420 to nqn.2016-06.io.spdk:cnode1: Successful
docker-compose run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 host add --subsystem "nqn.2016-06.io.spdk:cnode1" --host "*"
Allowing any host for nqn.2016-06.io.spdk:cnode1: Successful
```

#### Manual Steps

The same configuration can also be manually run:

1. First of all, let's create the `nvmeof-cli` shortcut to interact with the NVMe-oF gateway:
1. First of all, let's create the `cephnvmf` shortcut to interact with the NVMe-oF gateway:
```bash
eval $(make alias)
Expand All @@ -133,34 +122,28 @@ The same configuration can also be manually run:
make rbd
```
1. Create a bdev (Block Device) from an RBD image:
```bash
nvmeof-cli create_bdev --pool rbd --image demo_image --bdev demo_bdev
```
1. Create a subsystem:
```bash
nvmeof-cli create_subsystem --subnqn nqn.2016-06.io.spdk:cnode1 --serial SPDK00000000000001
cephnvmf subsystem add --subsystem nqn.2016-06.io.spdk:cnode1
```
1. Add a namespace:
```bash
nvmeof-cli add_namespace --subnqn nqn.2016-06.io.spdk:cnode1 --bdev demo_bdev
cephnvmf namespace add --subsystem nqn.2016-06.io.spdk:cnode1 --rbd-pool rbd --rbd-image demo_image
```
1. Create a listener so that NVMe initiators can connect to:
```bash
nvmeof-cli create_listener ---subnqn nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420
cephnvmf listener add ---subsystem nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420
```
1. Define which hosts can connect:
```bash
nvmeof-cli add_host --subnqn nqn.2016-06.io.spdk:cnode1 --host "*"
cephnvmf host add --subsystem nqn.2016-06.io.spdk:cnode1 --host "*"
```
Expand Down
Loading

0 comments on commit d447185

Please sign in to comment.