diff --git a/.gitignore b/.gitignore index 8b84efbf..1a1e4d55 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ qemu/bzImage subprojects/liburing-* subprojects/packagecache +subprojects/nlohmann_json-* diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..4b3c2a9d --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,14 @@ +# Configuring LSVD + +LSVD is configured using a JSON file. When creating an image, we will +try to read the following paths and parse them for configuration options: + +- Default built-in configuration +- `/usr/local/etc/lsvd.json` +- `./lsvd.json` +- user supplied path + +The file read last has highest priority. + +We will also first try to parse the user-supplied path as a JSON object, and if +that fails try treat it as a path and read it from a file. diff --git a/docs/example_config.json b/docs/example_config.json new file mode 100644 index 00000000..653a3e0f --- /dev/null +++ b/docs/example_config.json @@ -0,0 +1,18 @@ +{ + "rcache_dir": "/tmp/lsvd", + "rcache_size": 524288000, + "rcache_fetch_window": 12, + "wlog_dir": "/tmp/lsvd", + "wlog_bytes": 524288000, + "wlog_write_window": 8, + "wlog_chunk_bytes": 2097152, + "antithrash_ratio": 67, + "backend_obj_bytes": 8388608, + "backend_write_window": 8, + "checkpoint_interval_objs": 500, + "flush_timeout_ms": 2000, + "flush_interval_ms": 1000, + "gc_threshold_pc": 60, + "gc_write_window": 4, + "no_gc": false +} \ No newline at end of file diff --git a/meson.build b/meson.build index 8455276b..f3948564 100644 --- a/meson.build +++ b/meson.build @@ -30,6 +30,12 @@ executable( dependencies: lsvd_deps + [dependency('_spdk')], ) +executable( + 'lsvd_tgt', + lsvd_tgt, + dependencies: lsvd_deps + [dependency('_spdk')], +) + executable( 'imgtool', ['src/imgtool.cc'], diff --git a/src/bdev_lsvd.cc b/src/bdev_lsvd.cc index 8890edc6..661f7da1 100644 --- a/src/bdev_lsvd.cc +++ b/src/bdev_lsvd.cc @@ -2,7 +2,9 @@ #include "spdk/bdev_module.h" #include +#include "backend.h" #include "bdev_lsvd.h" +#include "config.h" #include "image.h" #include "request.h" #include "smartiov.h" @@ -217,15 +219,32 @@ int bdev_lsvd_create(str img_name, rados_ioctx_t ioctx, lsvd_config cfg) return 0; } -int bdev_lsvd_delete(std::string img_name) +int bdev_lsvd_create(str pool_name, str image_name, str user_cfg) { - auto p = std::promise(); - spdk_bdev_unregister_by_name( + auto be = connect_to_pool(pool_name); + auto cfg = lsvd_config::from_user_cfg(user_cfg); + PR_RET_IF(!cfg.has_value(), -1, "Failed to read config file"); + + return bdev_lsvd_create(image_name, be, cfg.value()); +} + +void bdev_lsvd_delete(str img_name, std::function cb) +{ + log_info("Deleting image '{}'", img_name); + auto rc = spdk_bdev_unregister_by_name( img_name.c_str(), &lsvd_if, + // some of the ugliest lifetime management code you'll ever see, but + // it should work [](void *arg, int rc) { - auto p = (std::promise *)arg; - p->set_value(rc); + log_info("Image deletion done, rc = {}", rc); + auto cb = (std::function *)arg; + (*cb)(rc); + delete cb; }, - &p); - return p.get_future().get(); + new std::function(cb)); + + if (rc != 0) { + log_error("Failed to delete image '{}': {}", img_name, rc); + cb(rc); + } } diff --git a/src/bdev_lsvd.h b/src/bdev_lsvd.h index 294298ce..3028494e 100644 --- a/src/bdev_lsvd.h +++ b/src/bdev_lsvd.h @@ -1,8 +1,10 @@ #pragma once #include +#include #include "config.h" int bdev_lsvd_create(str img_name, rados_ioctx_t io_ctx, lsvd_config cfg); -int bdev_lsvd_delete(str img_name); +int bdev_lsvd_create(str pool_name, str image_name, str cfg_path); +void bdev_lsvd_delete(str img_name, std::function cb); diff --git a/src/bdev_lsvd_rpc.cc b/src/bdev_lsvd_rpc.cc new file mode 100644 index 00000000..f9c80e67 --- /dev/null +++ b/src/bdev_lsvd_rpc.cc @@ -0,0 +1,104 @@ +#include "spdk/bdev.h" +#include "spdk/json.h" +#include "spdk/jsonrpc.h" +#include "spdk/likely.h" +#include "spdk/log.h" +#include "spdk/nvme.h" +#include "spdk/rpc.h" +#include "spdk/util.h" + +#include "bdev_lsvd.h" +#include "utils.h" + +/** + * We only expose 2 RPC endpoints: create and delete. Unlike RBD, we will not + * have commands to manage ceph clusters; each image will create its own. + */ + +struct rpc_create_lsvd { + char *image_name; + char *pool_name; + char *config; +}; + +static const struct spdk_json_object_decoder rpc_create_lsvd_decoders[] = { + {"image_name", offsetof(rpc_create_lsvd, image_name), + spdk_json_decode_string, false}, + {"pool_name", offsetof(rpc_create_lsvd, pool_name), spdk_json_decode_string, + false}, + {"config", offsetof(rpc_create_lsvd, config), spdk_json_decode_string, + true}, +}; + +static void rpc_bdev_lsvd_create(spdk_jsonrpc_request *req_json, + const spdk_json_val *params) +{ + std::unique_ptrimage_name); + free(p->pool_name); + free(p->config); + })> + req(new rpc_create_lsvd()); + + auto rc = spdk_json_decode_object(params, rpc_create_lsvd_decoders, + SPDK_COUNTOF(rpc_create_lsvd_decoders), + req.get()); + if (rc != 0) { + spdk_jsonrpc_send_error_response(req_json, rc, + "Failed to parse rpc json"); + return; + } + + rc = bdev_lsvd_create(req->pool_name, req->image_name, req->config); + if (rc != 0) { + spdk_jsonrpc_send_error_response(req_json, rc, + "Failed to create lsvd bdev"); + return; + } + + auto w = spdk_jsonrpc_begin_result(req_json); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(req_json, w); +} + +SPDK_RPC_REGISTER("bdev_lsvd_create", rpc_bdev_lsvd_create, SPDK_RPC_RUNTIME) + +struct rpc_delete_lsvd { + char *image_name; +}; + +static const struct spdk_json_object_decoder rpc_delete_lsvd_decoders[] = { + {"image_name", offsetof(rpc_delete_lsvd, image_name), + spdk_json_decode_string, false}, +}; + +static void rpc_bdev_lsvd_delete(struct spdk_jsonrpc_request *req_json, + const struct spdk_json_val *params) +{ + std::unique_ptrimage_name); })> + req(new rpc_delete_lsvd()); + + int rc = spdk_json_decode_object(params, rpc_delete_lsvd_decoders, + SPDK_COUNTOF(rpc_delete_lsvd_decoders), + req.get()); + if (rc != 0) { + spdk_jsonrpc_send_error_response(req_json, rc, + "Failed to parse rpc json"); + return; + } + + bdev_lsvd_delete(req->image_name, [=](int rc) { + if (rc == 0) { + auto w = spdk_jsonrpc_begin_result(req_json); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(req_json, w); + } else { + log_error("Failed to destroy lsvd bdev, rc = {}", rc); + spdk_jsonrpc_send_error_response(req_json, rc, + "Failed to destroy lsvd bdev"); + } + }); +} + +SPDK_RPC_REGISTER("bdev_lsvd_delete", rpc_bdev_lsvd_delete, SPDK_RPC_RUNTIME) diff --git a/src/config.cc b/src/config.cc index 04dfde18..8e8cac04 100644 --- a/src/config.cc +++ b/src/config.cc @@ -1,150 +1,129 @@ -/* - * file: config.cc - * description: quick and dirty config file parser - * env var overrides modeled on github.com/spf13/viper - * - * author: Peter Desnoyers, Northeastern University - * Copyright 2021, 2022 Peter Desnoyers - * license: GNU LGPL v2.1 or newer - * LGPL-2.1-or-later - */ - -#include +#include +#include +#include #include +#include #include #include -#include -#include -#include -#include -#include -namespace fs = std::filesystem; - #include "config.h" -#include "config_macros.h" +#include "utils.h" -vec cfg_path({"lsvd.conf", "/usr/local/etc/lsvd.conf"}); +lsvd_config lsvd_config::get_default() { return lsvd_config(); } -static void split(std::string s, vec &words) +opt lsvd_config::from_user_cfg(str cfg) { - std::string w = ""; - for (auto c : s) { - if (!isspace(c)) - w = w + c; - else { - words.push_back(w); - w = ""; + auto c = get_default(); + if (cfg.empty()) + return c; + + try { + c.parse_file("/usr/local/etc/lsvd.json", true); + c.parse_file("./lsvd.json", true); + + if (cfg[0] == '{') { + c.parse_json(cfg); + } else { + c.parse_file(cfg, false); } + return c; + } catch (const std::exception &e) { + log_error("Failed to parse config '{}': {}", cfg, e.what()); + return std::nullopt; } - if (w.size() > 0) - words.push_back(w); } -static std::map m = {{"file", BACKEND_FILE}, - {"rados", BACKEND_RADOS}}; - -/* fancy-ass macros to parse config file lines. - * config keyword = field name - * environment var = LSVD_ + uppercase(field name) - * skips blank lines and lines that don't match a keyword - */ +// https://stackoverflow.com/questions/116038/how-do-i-read-an-entire-file-into-a-stdstring-in-c +auto read_file(std::string_view path) -> std::string +{ + constexpr auto read_size = std::size_t(4096); + auto stream = std::ifstream(path.data()); + stream.exceptions(std::ios_base::badbit); + + if (not stream) + throw std::ios_base::failure("file does not exist"); + + auto out = std::string(); + auto buf = std::string(read_size, '\0'); + while (stream.read(&buf[0], read_size)) + out.append(buf, 0, stream.gcount()); + out.append(buf, 0, stream.gcount()); + return out; +} -int lsvd_config::read() +void lsvd_config::parse_file(str path, bool can_be_missing) { - auto explicit_cfg = getenv("LSVD_CONFIG_FILE"); - if (explicit_cfg) { - std::string f(explicit_cfg); - cfg_path.insert(cfg_path.begin(), f); - } - for (auto f : cfg_path) { - std::ifstream fp(f); - if (!fp.is_open()) - continue; - std::string line; - while (getline(fp, line)) { - if (line[0] == '#') - continue; - vec words; - split(line, words); - if (words.size() != 2) - continue; - F_CONFIG_H_INT(words[0], words[1], backend_obj_size); - F_CONFIG_INT(words[0], words[1], wcache_batch); - F_CONFIG_H_INT(words[0], words[1], wcache_chunk); - F_CONFIG_STR(words[0], words[1], rcache_dir); - F_CONFIG_STR(words[0], words[1], wcache_dir); - F_CONFIG_INT(words[0], words[1], num_parallel_writes); - F_CONFIG_TABLE(words[0], words[1], backend, m); - F_CONFIG_H_INT(words[0], words[1], cache_size); - F_CONFIG_H_INT(words[0], words[1], wlog_size); - F_CONFIG_INT(words[0], words[1], hard_sync); - F_CONFIG_INT(words[0], words[1], ckpt_interval); - F_CONFIG_INT(words[0], words[1], flush_timeout_msec); - F_CONFIG_INT(words[0], words[1], gc_threshold); - F_CONFIG_INT(words[0], words[1], fetch_window); - F_CONFIG_INT(words[0], words[1], fetch_ratio); - F_CONFIG_INT(words[0], words[1], no_gc); - F_CONFIG_INT(words[0], words[1], gc_window); + assert(!path.empty()); + if (access(path.c_str(), F_OK) == -1) { + if (can_be_missing) { + log_info("Optional config file missing: {}, continuing...", path); + return; } - fp.close(); - break; + log_error("Config file not found: {}", path); + return; } - ENV_CONFIG_H_INT(backend_obj_size); - ENV_CONFIG_INT(wcache_batch); - ENV_CONFIG_H_INT(wcache_chunk); - ENV_CONFIG_STR(rcache_dir); - ENV_CONFIG_STR(wcache_dir); - ENV_CONFIG_INT(num_parallel_writes); - ENV_CONFIG_TABLE(backend, m); - ENV_CONFIG_H_INT(cache_size); - ENV_CONFIG_H_INT(wlog_size); - ENV_CONFIG_INT(hard_sync); - ENV_CONFIG_INT(ckpt_interval); - ENV_CONFIG_INT(flush_timeout_msec); - ENV_CONFIG_INT(gc_threshold); - ENV_CONFIG_INT(fetch_window); - ENV_CONFIG_INT(fetch_ratio); - ENV_CONFIG_INT(no_gc); - ENV_CONFIG_INT(gc_window); - - return 0; // success + auto cfg = read_file(path); + parse_json(cfg); } -std::string lsvd_config::cache_filename(uuid_t &uuid, const char *name, - cfg_cache_type type) +#define JSON_GET_BOOL(key) \ + do { \ + if (js.contains(#key)) { \ + auto v = js[#key]; \ + if (v.is_boolean()) { \ + this->key = js[#key]; \ + } else { \ + log_error("Invalid value for key (must be bool): {}", #key); \ + } \ + } \ + } while (0) + +#define JSON_GET_UINT(key) \ + do { \ + if (js.contains(#key)) { \ + auto v = js[#key]; \ + if (v.is_number_unsigned()) { \ + this->key = js[#key]; \ + } else { \ + log_error("Invalid value for key (must be uint): {}", #key); \ + } \ + } \ + } while (0) + +#define JSON_GET_STR(key) \ + do { \ + if (js.contains(#key)) { \ + auto v = js[#key]; \ + if (v.is_string()) { \ + this->key = js[#key]; \ + } else { \ + log_error("Invalid value for key (must be str): {}", #key); \ + } \ + } \ + } while (0) + +void lsvd_config::parse_json(str json) { - char buf[256]; // PATH_MAX - std::string file(name); - file = fs::path(file).filename(); - const char *dir; - const char *f_ext; - - dir = (type == LSVD_CFG_READ) ? rcache_dir.c_str() : wcache_dir.c_str(); - f_ext = (type == LSVD_CFG_READ) ? "rcache" : "wcache"; - - sprintf(buf, "%s/%s.%s", dir, file.c_str(), f_ext); - if (access(buf, R_OK | W_OK) == 0) - return std::string((const char *)buf); - - char uuid_s[64]; - uuid_unparse(uuid, uuid_s); - sprintf(buf, "%s/%s.%s", dir, uuid_s, f_ext); - return std::string((const char *)buf); -} - -#if 0 -int main(int argc, char **argv) { - auto cfg = new lsvd_config; - cfg->read(); - - printf("batch: %d\n", cfg->batch_size); // h_int - printf("wc batch %d\n", cfg->wcache_batch); // int - printf("cache: %s\n", cfg->cache_dir.c_str()); // str - printf("backend: %d\n", (int)cfg->backend); // table - - uuid_t uu; - std::cout << cfg->cache_filename(uu, "foobar") << "\n"; + auto js = nlohmann::json::parse(json); + + JSON_GET_STR(rcache_dir); + JSON_GET_UINT(rcache_bytes); + JSON_GET_UINT(rcache_fetch_window); + + JSON_GET_STR(wlog_dir); + JSON_GET_UINT(wlog_bytes); + JSON_GET_UINT(wlog_write_window); + JSON_GET_UINT(wlog_chunk_bytes); + + JSON_GET_UINT(antithrash_ratio); + JSON_GET_UINT(backend_obj_bytes); + JSON_GET_UINT(backend_write_window); + JSON_GET_UINT(checkpoint_interval_objs); + JSON_GET_UINT(flush_timeout_ms); + JSON_GET_UINT(flush_interval_ms); + + JSON_GET_UINT(gc_threshold_pc); + JSON_GET_UINT(gc_write_window); + JSON_GET_BOOL(no_gc); } -#endif diff --git a/src/config.h b/src/config.h index 446f64cf..03b5efb3 100644 --- a/src/config.h +++ b/src/config.h @@ -1,56 +1,56 @@ #pragma once -/* - * file: config.h - * description: quick and dirty config file parser - * env var overrides modeled on github.com/spf13/viper - * - * author: Peter Desnoyers, Northeastern University - * Copyright 2021, 2022 Peter Desnoyers - * license: GNU LGPL v2.1 or newer - * LGPL-2.1-or-later - */ - -#include #include #include "utils.h" -enum cfg_backend { BACKEND_FILE = 1, BACKEND_RADOS = 2 }; - -enum cfg_cache_type { LSVD_CFG_READ = 1, LSVD_CFG_WRITE = 2 }; - class lsvd_config { public: - int backend_obj_size = 8 * 1024 * 1024; // in bytes - int wcache_batch = 8; // requests - int wcache_chunk = 2 * 1024 * 1024; // bytes - std::string rcache_dir = "/tmp/lsvd/"; - std::string wcache_dir = "/tmp/lsvd/"; - u32 num_parallel_writes = 8; - int hard_sync = 0; - enum cfg_backend backend = BACKEND_RADOS; - long cache_size = 500 * 1024 * 1024; // in bytes - long wlog_size = 500 * 1024 * 1024; // in bytes - int ckpt_interval = 500; // objects - int flush_timeout_msec = 2000; // flush timeout - int flush_interval_msec = 1000; // flush interval - int gc_threshold = 60; // GC threshold, percent - int gc_window = 4; // max GC writes outstanding - int fetch_window = 12; // read cache fetches - int fetch_ratio = 67; // anti-thrash served:backend ratio - int no_gc = 0; // turn off GC + // read cache + str rcache_dir = "/tmp/lsvd/"; // directory to store read cache files + u64 rcache_bytes = 500 * 1024 * 1024; // in bytes + u64 rcache_fetch_window = 12; // read cache fetches + + // write log + str wlog_dir = "/tmp/lsvd/"; + u64 wlog_bytes = 500 * 1024 * 1024; // in bytes + u64 wlog_write_window = 8; // requests + u64 wlog_chunk_bytes = 2 * 1024 * 1024; // bytes + + // backend + u64 antithrash_ratio = 67; // anti-thrash served:backend ratio + u64 backend_obj_bytes = 8 * 1024 * 1024; // in bytes + u64 backend_write_window = 8; // backend parallel writes + + u64 checkpoint_interval_objs = 500; // objects + u64 flush_timeout_ms = 2000; // flush timeout + u64 flush_interval_ms = 1000; // flush interval + + // GC + u64 gc_threshold_pc = 60; // GC threshold, percent + u64 gc_write_window = 4; // max GC writes outstanding + bool no_gc = false; // turn off GC - lsvd_config() {} ~lsvd_config() {} - int read(); - std::string cache_filename(uuid_t &uuid, const char *name, - cfg_cache_type type); inline fspath wlog_path(str imgname) { auto filename = imgname + ".wlog"; - return fspath(wcache_dir) / filename; + return fspath(wlog_dir) / filename; } + + /** + * Read LSVD configuration from user-supplied string. The string can be + * either a json string containing the configuration, the path to a file + * containing the same, or empty, in which case it will be ignored. + */ + static opt from_user_cfg(str cfg); + static lsvd_config get_default(); + + private: + lsvd_config() {} + + void parse_json(str js); + void parse_file(str path, bool can_be_missing = true); }; diff --git a/src/config_macros.h b/src/config_macros.h deleted file mode 100644 index 3b9eb88c..00000000 --- a/src/config_macros.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * file: config_macros.h - * description: really simple config file / env var parsing - * - * config file keyword = variable name - * environment variable = LSVD_(uppercase keyword) - * handles four types of values: - * - string - * - int - * - human-readable int (e.g. 10m, 20G) - * - table (actually std::map) lookup - * - * to use: - * F_CONFIG_XXX(input, arg, name) - if input==name, set name=arg - * ENV_CONFIG_XXX(name) - if LSVD_ is set, set name= - * - */ - -#include -#include -#include - -static long parseint(const char *_s) -{ - char *s = (char *)_s; - long val = strtol(s, &s, 0); - if (toupper(*s) == 'G') - val *= (1024 * 1024 * 1024); - if (toupper(*s) == 'M') - val *= (1024 * 1024); - if (toupper(*s) == 'K') - val *= 1024; - return val; -} - -static long parseint(std::string &s) { return parseint(s.c_str()); } - -#define CONFIG_HDR(name) \ - const char *val = NULL; \ - std::string env = "LSVD_" #name; \ - std::transform(env.begin(), env.end(), env.begin(), \ - [](unsigned char c) { return std::toupper(c); }); - -#define F_CONFIG_STR(input, arg, name) \ - { \ - if (input == #name) \ - name = arg; \ - } - -#define ENV_CONFIG_STR(name) \ - { \ - CONFIG_HDR(name) \ - if ((val = getenv(env.c_str()))) \ - name = std::string(val); \ - } - -#define F_CONFIG_INT(input, arg, name) \ - { \ - if (input == #name) \ - name = atoi(arg.c_str()); \ - } - -#define ENV_CONFIG_INT(name) \ - { \ - CONFIG_HDR(name) \ - if ((val = getenv(env.c_str()))) \ - name = atoi(val); \ - } - -#define F_CONFIG_H_INT(input, arg, name) \ - { \ - if (input == #name) \ - name = parseint(arg); \ - } - -#define ENV_CONFIG_H_INT(name) \ - { \ - CONFIG_HDR(name) \ - if ((val = getenv(env.c_str()))) \ - name = parseint(val); \ - } - -#define F_CONFIG_TABLE(input, arg, name, table) \ - { \ - if (input == #name) \ - name = table[arg]; \ - } - -#define ENV_CONFIG_TABLE(name, table) \ - { \ - CONFIG_HDR(name) \ - if ((val = getenv(env.c_str()))) \ - name = table[std::string(val)]; \ - } diff --git a/src/image.cc b/src/image.cc index 0db401ab..b92097ec 100644 --- a/src/image.cc +++ b/src/image.cc @@ -19,7 +19,8 @@ lsvd_image::lsvd_image(std::string name, rados_ioctx_t io, lsvd_config cfg_) : imgname(name), cfg(cfg_), io(io) { objstore = make_rados_backend(io); - rcache = get_read_cache_instance(cfg.rcache_dir, cfg.cache_size, objstore); + rcache = + get_read_cache_instance(cfg.rcache_dir, cfg.rcache_bytes, objstore); read_superblock(); debug("Found checkpoints: {}", checkpoints); @@ -165,7 +166,7 @@ seqnum_t lsvd_image::roll_forward_from_last_checkpoint() // This must be larger than the max backend batch size to avoid // potential corruption if subsequent breaks overlap with current dangling // objects and we get writes from two different "generations" - for (seqnum_t i = 1; i < cfg.num_parallel_writes * 4; i++) + for (seqnum_t i = 1; i < cfg.backend_write_window * 4; i++) objstore->delete_obj(oname(imgname, seq + i)); return seq; @@ -478,7 +479,7 @@ class lsvd_image::write_request : public lsvd_image::aio_request sector_t size_sectors = req_bytes / 512; // split large requests into 2MB (default) chunks - sector_t max_sectors = img->cfg.wcache_chunk / 512; + sector_t max_sectors = img->cfg.wlog_chunk_bytes / 512; n_req += div_round_up(req_bytes / 512, max_sectors); // TODO: this is horribly ugly diff --git a/src/lsvd_tgt.cc b/src/lsvd_tgt.cc new file mode 100644 index 00000000..aa7dfb4b --- /dev/null +++ b/src/lsvd_tgt.cc @@ -0,0 +1,29 @@ +#include "spdk/event.h" + +#include "utils.h" + +static void lsvd_tgt_usage() {} +static int lsvd_tgt_parse_arg(int ch, char *arg) { return 0; } + +static void lsvd_tgt_started(void *arg1) +{ + log_info("LSVD SPDK nvmf target started"); +} + +int main(int argc, char **argv) +{ + int rc; + struct spdk_app_opts opts = {}; + + spdk_app_opts_init(&opts, sizeof(opts)); + opts.name = "lsvd_tgt"; + if ((rc = spdk_app_parse_args(argc, argv, &opts, "", NULL, + lsvd_tgt_parse_arg, lsvd_tgt_usage)) != + SPDK_APP_PARSE_ARGS_SUCCESS) { + exit(rc); + } + + rc = spdk_app_start(&opts, lsvd_tgt_started, NULL); + spdk_app_fini(); + return rc; +} diff --git a/src/meson.build b/src/meson.build index 5af34b55..d43a21d3 100644 --- a/src/meson.build +++ b/src/meson.build @@ -29,6 +29,7 @@ lsvd_deps = [ dependency('boost', modules: ['system', 'filesystem', 'program_options', 'thread', 'regex']), dependency('liburing', static: true), dependency('uuid'), + dependency('nlohmann_json'), cxx.find_library('rados', required: true), cxx.find_library('jemalloc', required: false), ] @@ -36,4 +37,10 @@ lsvd_deps = [ spdk_fe = lsvd_src + files( 'bdev_lsvd.cc', 'spdk_frontend.cc', +) + +lsvd_tgt = lsvd_src + files( + 'lsvd_tgt.cc', + 'bdev_lsvd.cc', + 'bdev_lsvd_rpc.cc', ) \ No newline at end of file diff --git a/src/rpc_plugin.py b/src/rpc_plugin.py new file mode 100644 index 00000000..181dd52d --- /dev/null +++ b/src/rpc_plugin.py @@ -0,0 +1,27 @@ +from spdk.rpc.client import print_json + +def spdk_rpc_plugin_initialize(subparsers): + def bdev_lsvd_create(args): + print_json(args.client.call('bdev_lsvd_create', { + 'image_name': args.image_name, + 'pool_name': args.pool_name, + 'config': args.cfg or '' + })) + + p = subparsers.add_parser('bdev_lsvd_create', help='Create a bdev with LSVD backend') + p.add_argument('pool_name', help='Name of the ceph pool') + p.add_argument('image_name', help='Name of the lsvd disk image') + p.add_argument('-c', '--cfg', help='Path to config file OR inline JSON string', required=False) + p.set_defaults(func=bdev_lsvd_create) + + def bdev_lsvd_delete(args): + print_json(args.client.call('bdev_lsvd_delete', { + 'image_name': args.image_name + })) + + p = subparsers.add_parser('bdev_lsvd_delete', help='Delete a lsvd bdev') + p.add_argument('image_name', help='Name of the lsvd disk image') + p.set_defaults(func=bdev_lsvd_delete) + + + diff --git a/src/spdk_frontend.cc b/src/spdk_frontend.cc index 2b3034c6..5c07e8c1 100644 --- a/src/spdk_frontend.cc +++ b/src/spdk_frontend.cc @@ -181,8 +181,9 @@ static void start_lsvd(void *arg) auto trid = get_trid(HOSTNAME, PORT); // Add lsvd bdev - lsvd_config cfg; // TODO read this in from a config file - cfg.cache_size = 160 * 1024 * 1024; // small 160mb cache for testing + auto cfg = + lsvd_config::get_default(); // TODO read this in from a config file + cfg.rcache_bytes = 160 * 1024 * 1024; // small 160mb cache for testing auto err = bdev_lsvd_create(args->image_name, io_ctx, cfg); assert(err == 0); add_bdev_ns(nvme_ss, args->image_name); @@ -226,7 +227,6 @@ int main(int argc, const char **argv) return 1; } - auto args = (start_lsvd_args){ .pool_name = argv[1], .image_name = argv[2], @@ -238,7 +238,6 @@ int main(int argc, const char **argv) debug("Args: pool={}, image={}", args.pool_name, args.image_name); - spdk_app_opts opts = {.shutdown_cb = []() { log_info("Shutting down LSVD SPDK program ..."); spdk_app_stop(0); diff --git a/src/spdk_wrap.cc b/src/spdk_wrap.cc index d7d90941..71621f90 100644 --- a/src/spdk_wrap.cc +++ b/src/spdk_wrap.cc @@ -1,6 +1,6 @@ #include "spdk_wrap.h" #include "config.h" -#include "src/utils.h" +#include "utils.h" spdk_completion::spdk_completion(rbd_callback_t cb, void *cb_arg) : cb(cb), cb_arg(cb_arg) @@ -59,10 +59,7 @@ inline void spdk_completion::dec_and_free() lsvd_rbd *lsvd_rbd::open_image(rados_ioctx_t io, std::string name) { try { - lsvd_config cfg; - auto err = cfg.read(); - PR_ERR_RET_IF(err < 0, nullptr, -err, "Failed to read config"); - + auto cfg = lsvd_config::get_default(); return new lsvd_rbd(name, io, cfg); } catch (std::runtime_error &e) { log_error("Failed to open image: {}", e.what()); diff --git a/src/translate.cc b/src/translate.cc index 0806980c..00b83281 100644 --- a/src/translate.cc +++ b/src/translate.cc @@ -13,8 +13,8 @@ #include "extent.h" #include "misc_cache.h" #include "request.h" -#include "src/utils.h" #include "translate.h" +#include "utils.h" /* * Architecture: @@ -237,11 +237,11 @@ class translate_impl : public translate total_live_sectors += oi.live; } - current = new translate_req(REQ_PUT, cfg.backend_obj_size, this); + current = new translate_req(REQ_PUT, cfg.backend_obj_bytes, this); assert(current->batch_buf != nullptr); // start worker, flush, and GC threads - if (cfg.flush_interval_msec > 0) + if (cfg.flush_interval_ms > 0) flush_worker = std::jthread([this](std::stop_token st) { flush_thread(st); }); @@ -400,7 +400,7 @@ ssize_t translate_impl::writev(uint64_t cache_seq, size_t offset, iovec *iov, std::unique_lock lk(m); if (!current->room(bytes)) { workers->put_locked(current); - current = new translate_req(REQ_PUT, cfg.backend_obj_size, this); + current = new translate_req(REQ_PUT, cfg.backend_obj_bytes, this); } // write the data into the in-memory log @@ -459,7 +459,7 @@ ssize_t translate_impl::trim(size_t offset, size_t len) void translate_impl::backend_backpressure(void) { std::unique_lock lk(m); - while (outstanding_writes > cfg.num_parallel_writes) + while (outstanding_writes > cfg.backend_write_window) cv.wait(lk); } @@ -643,7 +643,7 @@ void translate_impl::write_gc(seqnum_t _seq, translate_req *req) data_sectors += e.len; int max_hdr_bytes = sizeof(common_obj_hdr) + sizeof(obj_data_hdr) + - (cfg.backend_obj_size / 2048) * sizeof(data_map); + (cfg.backend_obj_bytes / 2048) * sizeof(data_map); int max_hdr_sectors = div_round_up(max_hdr_bytes, 512); auto buf = req->gc_buf = @@ -855,7 +855,7 @@ void translate_impl::flush(void) if (current->len > 0) { workers->put_locked(current); - current = new translate_req(REQ_PUT, cfg.backend_obj_size, this); + current = new translate_req(REQ_PUT, cfg.backend_obj_bytes, this); } auto flush_req = new translate_req(REQ_FLUSH, this); @@ -870,7 +870,7 @@ void translate_impl::checkpoint(void) if (current->len > 0) { workers->put_locked(current); - current = new translate_req(REQ_PUT, cfg.backend_obj_size, this); + current = new translate_req(REQ_PUT, cfg.backend_obj_bytes, this); } auto ckpt_req = new translate_req(REQ_CKPT, this); @@ -886,8 +886,8 @@ void translate_impl::checkpoint(void) void translate_impl::flush_thread(std::stop_token st) { pthread_setname_np(pthread_self(), "flush_thread"); - auto interval = std::chrono::milliseconds(cfg.flush_interval_msec); - auto timeout = std::chrono::milliseconds(cfg.flush_timeout_msec); + auto interval = std::chrono::milliseconds(cfg.flush_interval_ms); + auto timeout = std::chrono::milliseconds(cfg.flush_timeout_ms); auto t0 = std::chrono::system_clock::now(); auto seq0 = cur_seq.load(); @@ -902,7 +902,7 @@ void translate_impl::flush_thread(std::stop_token st) if (std::chrono::system_clock::now() - t0 < timeout) continue; workers->put_locked(current); - current = new translate_req(REQ_PUT, cfg.backend_obj_size, this); + current = new translate_req(REQ_PUT, cfg.backend_obj_bytes, this); } else { seq0 = cur_seq.load(); t0 = std::chrono::system_clock::now(); @@ -993,7 +993,7 @@ void translate_impl::do_gc(std::stop_token &st) /* gather list of objects needing cleaning, return if none */ - const double threshold = cfg.gc_threshold / 100.0; + const double threshold = cfg.gc_threshold_pc / 100.0; vec> objs_to_clean; for (auto [u, o, n] : utilization) { if (u > threshold) @@ -1076,7 +1076,7 @@ void translate_impl::do_gc(std::stop_token &st) std::queue requests; while (all_extents.size() > 0) { - sector_t sectors = 0, max = cfg.backend_obj_size / 512; + sector_t sectors = 0, max = cfg.backend_obj_bytes / 512; vec<_extent> extents; auto it = all_extents.begin(); @@ -1111,7 +1111,7 @@ void translate_impl::do_gc(std::stop_token &st) workers->put_locked(req); lk.unlock(); - while ((int)requests.size() > cfg.gc_window && + while (requests.size() > cfg.gc_write_window && !st.stop_requested()) { auto t = requests.front(); t->wait(); diff --git a/src/utils.h b/src/utils.h index 66733c59..63d9daa9 100644 --- a/src/utils.h +++ b/src/utils.h @@ -124,6 +124,14 @@ using fspath = std::filesystem::path; } \ } while (0) +#define PR_GOTO_IF(cond, lbl, MSG, ...) \ + do { \ + if (cond) { \ + log_error(MSG, ##__VA_ARGS__); \ + goto lbl; \ + } \ + } while (0) + /** * If `cond` is true, print an error message to stdout with MSG, then return * `ret` diff --git a/src/write_cache.cc b/src/write_cache.cc index a4d22aa0..ffcceca7 100644 --- a/src/write_cache.cc +++ b/src/write_cache.cc @@ -474,7 +474,7 @@ write_cache_impl::write_cache_impl(int fd, translate &be, lsvd_config &cfg) int n_pages = super->limit - super->base; max_write_pages = n_pages / 2 + n_pages / 4; // no idea why this is 3/4ths - write_batch = cfg.wcache_batch; + write_batch = cfg.wlog_write_window; } write_cache_impl::~write_cache_impl() @@ -580,7 +580,7 @@ uptr open_wlog(fspath path, translate &xlate, lsvd_config &cfg) fd = open(path.c_str(), O_RDWR | O_CREAT, 0644); PR_ERR_RET_IF(fd < 0, nullptr, errno, "Failed to create cache file"); - auto err = init_wcache(fd, xlate.uuid, cfg.wlog_size); + auto err = init_wcache(fd, xlate.uuid, cfg.wlog_bytes); PR_ERR_RET_IF(err < 0, nullptr, -err, "Failed to init wlog"); } else { fd = open(path.c_str(), O_RDWR); diff --git a/subprojects/nlohmann_json.wrap b/subprojects/nlohmann_json.wrap new file mode 100644 index 00000000..bf5d7000 --- /dev/null +++ b/subprojects/nlohmann_json.wrap @@ -0,0 +1,10 @@ +[wrap-file] +directory = nlohmann_json-3.11.2 +lead_directory_missing = true +source_url = https://github.com/nlohmann/json/releases/download/v3.11.2/include.zip +source_filename = nlohmann_json-3.11.2.zip +source_hash = e5c7a9f49a16814be27e4ed0ee900ecd0092bfb7dbfca65b5a421b774dccaaed +wrapdb_version = 3.11.2-1 + +[provide] +nlohmann_json = nlohmann_json_dep diff --git a/subprojects/packagefiles/spdk/meson.build b/subprojects/packagefiles/spdk/meson.build index c798d74c..8c57dbfe 100644 --- a/subprojects/packagefiles/spdk/meson.build +++ b/subprojects/packagefiles/spdk/meson.build @@ -50,6 +50,7 @@ custom_libnames = [ # 'spdk_bdev_split', # 'spdk_bdev_delay', # 'spdk_bdev_zone_block', + # 'spdk_bdev_rbd', 'spdk_blobfs_bdev', 'spdk_blobfs', 'spdk_blob_bdev',