From e53de875211852dbfc08f6cb54a5015d21406f5a Mon Sep 17 00:00:00 2001 From: Alex Reid Date: Sun, 7 Aug 2022 11:28:59 +0100 Subject: [PATCH] New feature - option to use libtcmu without netlink Some folks might be using libtcmu in their own applications which don't hook into TCMU runner, much like the consumer.c example. In those cases, it can be useful to avoid using netlink altogether. As I'm sure you are aware, it's almost impossible to have more than one user of TCMU when netlink is involved. This is because the kernel broadcasts device events to all TCMU listeners over netlink and simply accepts the first response it receives. This can lead to the following scenario: 1. Application A creates a TCMU backstore with handler=app_a 2. The kernel (target_core_user) broadcasts the netlink message to all instances of libtcmu 3. Application B receives the message first. Application B doesn't have a handler for app_a, so it replies with an error code to the kernel. 4. The kernel receives the error and aborts backstore creation. 5. Application A wonders why its backstore creation failed. Users can avoid using netlink altogether by setting nl_reply_supported=-1 at backing store creation time. This is great as multiple applications using TCMU can now coexist on the same machine. The problem is we don't have a nice way to utilise libtcmu in this model. We need a way to notify libtcmu that a device has been created, reconfigured, removed so the relevant handlers can be called. Currently, this is done via netlink. This PR allows users to utilise all the great features of libtcmu, but without using netlink. The PR makes the following changes: * modify tcmulib_initialize to take a boolean parameter which informs libtcmu whether or not it should set up netlink. If true we do. If false we don't. * expose new methods tcmulib_notify_device_added, tcmulib_notify_device_removed and tcmulib_notify_device_reconfiged, which application code should call to inform libtcmu that a device has been added, removed, or reconfigured. These API's take the dev_name and pass through to the existing device_add, device_remove and device_reconfig functions in libtcmu.c. tcmulib_notify_device_reconfiged will need additional parameters to specify the new config. * updates the documentation to reflect these changes and notes that tcmulib_master_fd_ready and tcmulib_get_master_fd should not be called if you chose to call tcmulib_initialize in "don't use netlink mode". * adds an example application consumer_no_netlink which shows how to create a standalone application using libtcmu without netlink. Issue #678. PR #694 Signed-off-by: Xiubo Li --- CMakeLists.txt | 7 + consumer.c | 3 +- consumer_no_netlink.c | 299 ++++++++++++++++++++++++++++++++++++++++++ create_backstore.sh | 17 +++ delete_backstore.sh | 6 + find_uio_device.sh | 15 +++ libtcmu.c | 62 +++++++-- libtcmu.h | 48 ++++++- main.c | 2 +- tcmu-synthesizer.c | 2 +- 10 files changed, 446 insertions(+), 15 deletions(-) create mode 100644 consumer_no_netlink.c create mode 100755 create_backstore.sh create mode 100755 delete_backstore.sh create mode 100755 find_uio_device.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 9276f4d8..de6ff7a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,13 @@ add_executable(consumer ) target_link_libraries(consumer tcmu) +# As per `consumer`, but demonstrates how to use libtcmu without netlink +add_executable(consumer_no_netlink + scsi.c + consumer_no_netlink.c + ) + +target_link_libraries(consumer_no_netlink tcmu) if (with-zbc) # Stuff for building the file zbc handler add_library(handler_file_zbc diff --git a/consumer.c b/consumer.c index f5167d15..9fa1059e 100644 --- a/consumer.c +++ b/consumer.c @@ -130,6 +130,7 @@ int main(int argc, char **argv) struct pollfd pollfds[16]; int i; int ret; + bool use_netlink = true; if (tcmu_setup_log(LOG_DIR)) { fprintf(stderr, "Could not setup tcmu logger.\n"); @@ -139,7 +140,7 @@ int main(int argc, char **argv) /* If any TCMU devices that exist that match subtype, handler->added() will now be called from within tcmulib_initialize(). */ - tcmulib_ctx = tcmulib_initialize(&foo_handler, 1); + tcmulib_ctx = tcmulib_initialize(&foo_handler, 1, use_netlink); if (!tcmulib_ctx) { tcmu_err("tcmulib_initialize failed with %p\n", tcmulib_ctx); exit(1); diff --git a/consumer_no_netlink.c b/consumer_no_netlink.c new file mode 100644 index 00000000..8a6db475 --- /dev/null +++ b/consumer_no_netlink.c @@ -0,0 +1,299 @@ +/* + * This file is licensed to you under your choice of the GNU Lesser + * General Public License, version 2.1 or any later version (LGPLv2.1 or + * later), or the Apache License 2.0. + */ + +/* + * This example shows how one can use TCMU without using netlink. Only kernels + * which support per device disabling of netlink in target_core_user may use + * this feature. To check whether your kernel supports this feature try and + * create a TCMU backstore with `nl_reply_supported` set to "-1": + * + * $ mkdir -p /sys/kernal/config/target/core/user_1234/test + * $ echo "dev_size=1048576,dev_config=foo/bar,nl_reply_supported=-1" > /sys/kernel/config/target/core/user_1234/test/control + * $ echo 1 > /sys/kernel/config/target/core/user_1234/test/enable + * + * Using TCMU without netlink is useful for scenarios where they may be + * multiple applications using TCMU on one host. This sort of configuration is + * not possible when TCMU uses netlink because backstore device events are + * broadcasted over netlink to all TCMU subscribers. This mean one application + * can receive netlink events for devices which pertain to another application. + * If an app receives a netlink event for a device it does not know about libtcmu + * will return an error message and the kernel will fail the device action. + * + * The example creates a TCMU backing store (using the create_backstore.sh + * script) and then notifies libtcmu that the backstore has been created using + * tcmulib_notify_device_added. The example then sits in a loop processing + * commands. Note: unless you set up an HBA and LUN there won't be any + * commands to process. When the app recieves a termination signal it exits + * the processing loop. It then notifies libtcmu that the backstore is about to + * be removed via the tcmulib_notify_device_removed method. We then delete the + * backstore using the remove_backstore.sh script. + * + * This example must be ran in the the same directory as its accompanying + * bash scripts, i.e. the top level repo directory. + * + * NOTE: TCMU without netlink is not a panacea to the multi-TCMU problem. One + * still has to be careful to ensure that multiple apps don't try and create + * the same backstore (user_XXXX) at the same time. It may be necessary to + * synchronise/serialise other configs actions too. + * + * NOTE: this is a bare-bones example! It eschews any proper error handling, + * uses hardcoded values, and relies on bash scripts. It's intended to give + * a rough outline of how to use the feature and should not be used as the + * basis of any production code. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libtcmu.h" +#include "libtcmu_log.h" +#include "scsi.h" +#include "scsi_defs.h" +#include "target_core_user_local.h" + +#define LOG_DIR "/var/log" +#define CFG_STRING "tcm-user/1/test/foo/xxx" + +struct tcmu_device *tcmu_dev_array[128]; +size_t dev_array_len = 0; + +struct foo_state { + int fd; + uint64_t num_lbas; + uint32_t block_size; +}; + +static volatile int keep_running = 1; + +static void signal_handler(int) { + keep_running = 0; +} + +static bool run_command(char* buffer, size_t size, char* cmd) +{ + FILE *f = popen(cmd, "r"); + if (!f) { + tcmu_err("couldn't run command %s\n", cmd); + return false; + } + + fgets(buffer, size, f); + pclose(f); + return true; +} + +static bool create_backstore() +{ + int ret = system("./create_backstore.sh"); + return ret == 0; +} + +static bool find_uio_device(char** uio_name) +{ + char buffer[1024]; + if (!run_command(buffer, sizeof(buffer), "./find_uio_device.sh")) + return false; + + *uio_name = strdup(buffer); + return true; +} + +static void delete_backstore() +{ + system("./delete_backstore.sh"); +} + +static int foo_open(struct tcmu_device *dev) +{ + /* open the backing file */ + /* alloc private struct 'foo_state' */ + /* Save a ptr to it in dev->hm_private */ + + /* Add new device to our horrible fixed-length array */ + tcmu_dev_array[dev_array_len] = dev; + dev_array_len++; + + return 0; +} + +static void foo_close(struct tcmu_device *dev) +{ + /* not supported in this example */ +} + +static int foo_handle_cmd( + struct tcmu_device *dev, + uint8_t *cdb, + struct iovec *iovec, + size_t iov_cnt, + uint8_t *sense) +{ + struct foo_state *state = tcmu_dev_get_private(dev); + uint8_t cmd; + + cmd = cdb[0]; + + switch (cmd) { + case INQUIRY: + return tcmu_emulate_inquiry(dev, NULL, cdb, iovec, iov_cnt); + case TEST_UNIT_READY: + return tcmu_emulate_test_unit_ready(cdb, iovec, iov_cnt); + case SERVICE_ACTION_IN_16: + if (cdb[1] == READ_CAPACITY_16) + return tcmu_emulate_read_capacity_16(state->num_lbas, + state->block_size, + cdb, iovec, iov_cnt); + else + return TCMU_STS_NOT_HANDLED; + break; + case MODE_SENSE: + case MODE_SENSE_10: + return tcmu_emulate_mode_sense(dev, cdb, iovec, iov_cnt); + break; + case MODE_SELECT: + case MODE_SELECT_10: + return tcmu_emulate_mode_select(dev, cdb, iovec, iov_cnt); + break; + case READ_6: + case READ_10: + case READ_12: + case READ_16: + // A real "read" implementation goes here! + return TCMU_STS_RD_ERR; + + case WRITE_6: + case WRITE_10: + case WRITE_12: + case WRITE_16: + // A real "write" implemention goes here! + return TCMU_STS_OK; + + default: + tcmu_err("unknown command %x\n", cdb[0]); + return TCMU_STS_NOT_HANDLED; + } +} + +static struct tcmulib_handler foo_handler = { + .name = "Handler for foo devices (example code)", + .subtype = "foo", + .cfg_desc = "a description goes here", + + .added = foo_open, + .removed = foo_close, +}; + +int main(int argc, char **argv) +{ + struct tcmulib_context *tcmulib_ctx; + struct pollfd pollfds[16]; + int i; + int ret; + bool use_netlink = false; + char* uio_name = NULL; + + signal(SIGINT, signal_handler); + + if (tcmu_setup_log(LOG_DIR)) { + fprintf(stderr, "Could not setup tcmu logger.\n"); + exit(1); + } + + /* If any TCMU devices that exist that match subtype, + handler->added() will now be called from within + tcmulib_initialize(). */ + tcmulib_ctx = tcmulib_initialize(&foo_handler, 1, use_netlink); + if (!tcmulib_ctx) { + tcmu_err("tcmulib_initialize failed with %p\n", tcmulib_ctx); + exit(1); + } + + /* Create a TCMU backstore */ + if (!create_backstore()) { + tcmu_err("failed to create backstore\n"); + exit(1); + } + + /* Find the uio device */ + if (!find_uio_device(&uio_name)) { + tcmu_err("failed to find uio device for backstore\n"); + goto cleanup1; + } + + /* Notify libtcmu that we've created the device */ + ret = tcmulib_notify_device_added(tcmulib_ctx, uio_name, CFG_STRING); + if (ret != 0) { + tcmu_err("tcmulib_notify_device_added failed\n"); + goto cleanup2; + } + + while (keep_running) { + for (i = 0; i < dev_array_len; i++) { + pollfds[i].fd = tcmu_dev_get_fd(tcmu_dev_array[i]); + pollfds[i].events = POLLIN; + pollfds[i].revents = 0; + } + + ret = ppoll(pollfds, dev_array_len, NULL, NULL); + if (ret == -1 && keep_running) { + tcmu_err("ppoll() returned %d, exiting\n", ret); + exit(EXIT_FAILURE); + } + + /* Process any commands - in this demo binary there won't be any commands + to process unless you set up a HBA and LUN for the backstore */ + for (i = 0; i < dev_array_len; i++) { + if (pollfds[i].revents) { + struct tcmulib_cmd *cmd; + struct tcmu_device *dev = tcmu_dev_array[i]; + + tcmulib_processing_start(dev); + + while ((cmd = tcmulib_get_next_command(dev, 0)) != NULL) { + ret = foo_handle_cmd(dev, + cmd->cdb, + cmd->iovec, + cmd->iov_cnt, + cmd->sense_buf); + tcmulib_command_complete(dev, cmd, ret); + } + + tcmulib_processing_complete(dev); + } + } + } + + tcmu_info("main thread exiting\n"); + +cleanup2: + /* Notify the library that the device is about to be removed */ + tcmu_info("calling tcmulib_notify_device_removed\n"); + tcmulib_notify_device_removed(tcmulib_ctx, uio_name); + +cleanup1: + /* Delete the backstore */ + tcmu_info("removing backstore\n"); + delete_backstore(); + + free(uio_name); + + return 0; +} diff --git a/create_backstore.sh b/create_backstore.sh new file mode 100755 index 00000000..4b508a50 --- /dev/null +++ b/create_backstore.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +user_dir="user_1" +dev_config="foo/xxx" + +if [ -d /sys/kernel/config/target/core/$user_dir/test ]; then + echo "backstore already exists - remove it with delete_backstore.sh" + exit 1 +fi + +mkdir -p /sys/kernel/config/target/core/$user_dir/test +echo "dev_size=1048576,hw_max_sectors=256,hw_block_size=4096,dev_config=$dev_config,nl_reply_supported=-1" > /sys/kernel/config/target/core/$user_dir/test/control +echo "1" > /sys/kernel/config/target/core/$user_dir/test/enable +echo "created backstore" +exit 0 diff --git a/delete_backstore.sh b/delete_backstore.sh new file mode 100755 index 00000000..2932376c --- /dev/null +++ b/delete_backstore.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +user_dir="user_1" + +rmdir /sys/kernel/config/target/core/$user_dir/test +rmdir /sys/kernel/config/target/core/$user_dir diff --git a/find_uio_device.sh b/find_uio_device.sh new file mode 100755 index 00000000..f9878036 --- /dev/null +++ b/find_uio_device.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Finds the name of the uio device pertaining to the TCMU backstore +# "test/foo". The uio device - e.g. uio0 - is printed to stdout. + +for d in /sys/class/uio/*/name; +do + cfgstring=`cat $d` + if [[ $cfgstring == tcm-user/*/test/foo/xxx ]]; then + uio=`echo $d | grep -oP "uio\d"` + echo -n $uio + exit 0 + fi +done +exit 1 diff --git a/libtcmu.c b/libtcmu.c index 46a6fbcc..fcabc17e 100644 --- a/libtcmu.c +++ b/libtcmu.c @@ -767,7 +767,9 @@ static int open_devices(struct tcmulib_context *ctx) static void release_resources(struct tcmulib_context *ctx) { - teardown_netlink(ctx->nl_sock); + if (ctx->nl_sock) + teardown_netlink(ctx->nl_sock); + darray_free(ctx->handlers); darray_free(ctx->devices); free(ctx); @@ -775,7 +777,8 @@ static void release_resources(struct tcmulib_context *ctx) struct tcmulib_context *tcmulib_initialize( struct tcmulib_handler *handlers, - size_t handler_count) + size_t handler_count, + bool use_netlink) { struct tcmulib_context *ctx; int ret; @@ -785,10 +788,12 @@ struct tcmulib_context *tcmulib_initialize( if (!ctx) return NULL; - ctx->nl_sock = setup_netlink(ctx); - if (!ctx->nl_sock) { - free(ctx); - return NULL; + if (use_netlink) { + ctx->nl_sock = setup_netlink(ctx); + if (!ctx->nl_sock) { + free(ctx); + return NULL; + } } darray_init(ctx->handlers); @@ -817,12 +822,18 @@ void tcmulib_close(struct tcmulib_context *ctx) int tcmulib_get_master_fd(struct tcmulib_context *ctx) { - return nl_socket_get_fd(ctx->nl_sock); + if (ctx->nl_sock) + return nl_socket_get_fd(ctx->nl_sock); + else + return -1; } int tcmulib_master_fd_ready(struct tcmulib_context *ctx) { - return nl_recvmsgs_default(ctx->nl_sock); + if (ctx->nl_sock) + return nl_recvmsgs_default(ctx->nl_sock); + else + return -1; } void *tcmu_dev_get_private(struct tcmu_device *dev) @@ -1373,3 +1384,38 @@ void tcmulib_processing_complete(struct tcmu_device *dev) tcmu_err("failed to write device /dev/%s, %d\n", dev->dev_name, errno); } + +int tcmulib_notify_device_reconfig(struct tcmulib_context* ctx, char* dev_name, struct tcmulib_cfg_info* cfg) +{ + struct tcmu_device *dev; + int i, ret; + + dev = lookup_dev_by_name(ctx, dev_name, &i); + if (!dev) { + tcmu_err("Could not reconfigure device %s: not found.\n", + dev_name); + return -ENODEV; + } + + if (!dev->handler->reconfig) { + tcmu_dev_dbg(dev, "Reconfiguration is not supported with this device.\n"); + return -EOPNOTSUPP; + } + + ret = dev->handler->reconfig(dev, cfg); + if (ret < 0) { + tcmu_dev_dbg(dev, "Handler reconfig for %s failed with error %s.\n", + tcmulib_cfg_type_lookup[cfg->type], strerror(-ret)); + } + + return ret; +} + +int tcmulib_notify_device_added(struct tcmulib_context *ctx, char *dev_name, + char *cfgstring) { + return device_add(ctx, dev_name, cfgstring, false); +} + +void tcmulib_notify_device_removed(struct tcmulib_context *ctx, char *dev_name) { + device_remove(ctx, dev_name, false); +} diff --git a/libtcmu.h b/libtcmu.h index 8ac62329..bcc9482e 100644 --- a/libtcmu.h +++ b/libtcmu.h @@ -84,24 +84,64 @@ struct tcmulib_handler { /* Opaque (private) type */ struct tcmulib_context; -/* Claim subtypes you wish to handle. Returns libtcmu's master fd or -error.*/ +/* + * Claim subtypes you wish to handle. Returns a tcmulib_context or a NULL + * pointer on error. If you wish use libtcmu without netlink set use_netlink=false. + * In this mode you must: + * * create all backstores with netlink disabled (by setting nl_reply_supported=-1 + * at backstore create time) + * * manually notify the library of backstore creation, reconfiguration and removal + * events via the tcmu_notify* APIs. + */ struct tcmulib_context *tcmulib_initialize( struct tcmulib_handler *handlers, - size_t handler_count); + size_t handler_count, + bool use_netlink); /* Register to TCMU DBus service, for the claimed subtypes to be configurable * in targetcli. */ void tcmulib_register(struct tcmulib_context *ctx); -/* Gets the master file descriptor used by tcmulib. */ +/* Gets the master file descriptor used by tcmulib. If you called tcmulib_initialize + * with use_netlink=false then we aren't using netlink for device notifications and + * you shouldn't use this method. + */ int tcmulib_get_master_fd(struct tcmulib_context *ctx); /* * Call this when the master fd becomes ready, from your main thread. - * Handlers' callbacks may be called before it returns. + * Handlers' callbacks may be called before it returns. If you called + * tcmulib_initialize with use_netlink=false then we aren't using netlink + * for device notifications and you shouldn't use this method. */ int tcmulib_master_fd_ready(struct tcmulib_context *ctx); +/* + * Only applicable if tcmulib_initialize was called with use_netlink=false + * + * Notify the library that a TCMU backstore has been created. This + * function will open the backstore and call the appropriate handler. + */ +int tcmulib_notify_device_added(struct tcmulib_context *ctx, char *dev_name, + char *cfgstring); + +/* + * Only applicable if tcmulib_initialize was called with use_netlink=false + * + * Notify the library that the TCMU backstore is about to be removed. + * This function will close the backstore and call the removed handler. + * This method should be called before deleting the backstore. + */ +void tcmulib_notify_device_removed(struct tcmulib_context *ctx, char *dev_name); + +/* + * Only applicable if tcmulib_initialize was called with use_netlink=false + * + * Notify the library that the TCMU backstore should be reconfigured. + * This function will simply call the reconfig handler. + */ +int tcmulib_notify_device_reconfig(struct tcmulib_context* ctx, char* dev_name, struct tcmulib_cfg_info *cfg); + /* * When a device fd becomes ready, call this to get SCSI cmd info in * 'cmd' struct. libtcmu will allocate hm_cmd_size bytes for each cmd diff --git a/main.c b/main.c index 70b42335..97f6d57d 100644 --- a/main.c +++ b/main.c @@ -1380,7 +1380,7 @@ int main(int argc, char **argv) darray_append(handlers, tmp_handler); } - tcmulib_context = tcmulib_initialize(handlers.item, handlers.size); + tcmulib_context = tcmulib_initialize(handlers.item, handlers.size, true); if (!tcmulib_context) { tcmu_err("tcmulib_initialize failed\n"); goto err_free_handlers; diff --git a/tcmu-synthesizer.c b/tcmu-synthesizer.c index a9f7cb8e..2ed62afb 100644 --- a/tcmu-synthesizer.c +++ b/tcmu-synthesizer.c @@ -197,7 +197,7 @@ int main(int argc, char **argv) exit(1); } - ctx = tcmulib_initialize(&syn_handler, 1); + ctx = tcmulib_initialize(&syn_handler, 1, true); if (!ctx) { tcmu_err("tcmulib_initialize failed\n"); tcmu_destroy_log();