From 3bd79f8b28e3ff2ad8ed63f2c5594a1670803ed5 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Mon, 12 Feb 2024 09:31:05 +0100 Subject: [PATCH] cdefs: refactor to support version 3.0.X This patch perform the following: - refactor the code to be able to use libyang 3.0.X - adds additional parsing options Signed-off-by: Stefan Gula --- cffi/cdefs.h | 46 ++++++++++------ cffi/source.c | 8 +-- libyang/context.py | 19 ++++--- libyang/data.py | 131 +++++++++++++++++++++++++++++++++++++-------- libyang/log.py | 21 +++++--- tests/test_data.py | 16 +++--- 6 files changed, 176 insertions(+), 65 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 304ae55d..4b5736f3 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -179,10 +179,10 @@ int ly_log_options(int); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *); -void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *), int); -struct ly_err_item *ly_err_first(const struct ly_ctx *); +void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t)); +const struct ly_err_item *ly_err_first(const struct ly_ctx *); +const struct ly_err_item *ly_err_last(const struct ly_ctx *); void ly_err_clean(struct ly_ctx *, struct ly_err_item *); -LY_VECODE ly_vecode(const struct ly_ctx *); #define LYS_UNKNOWN ... #define LYS_CONTAINER ... @@ -238,14 +238,15 @@ struct lysc_node { struct ly_err_item { LY_LOG_LEVEL level; - LY_ERR no; + LY_ERR err; LY_VECODE vecode; char *msg; - char *path; + char *data_path; + char *schema_path; + uint64_t line; char *apptag; struct ly_err_item *next; struct ly_err_item *prev; - ...; }; struct lyd_node { @@ -261,11 +262,15 @@ struct lyd_node { LY_ERR lys_set_implemented(struct lys_module *, const char **); +#define LYD_NEW_VAL_OUTPUT ... +#define LYD_NEW_VAL_STORE_ONLY ... +#define LYD_NEW_VAL_BIN ... +#define LYD_NEW_VAL_CANON ... +#define LYD_NEW_META_CLEAR_DFLT ... #define LYD_NEW_PATH_UPDATE ... -#define LYD_NEW_PATH_OUTPUT ... -#define LYD_NEW_PATH_OPAQ ... -#define LYD_NEW_PATH_BIN_VALUE ... -#define LYD_NEW_PATH_CANON_VALUE ... +#define LYD_NEW_PATH_OPAQ ... +#define LYD_NEW_PATH_WITH_OPAQ ... +#define LYD_NEW_ANY_USE_VALUE ... LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **); LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **); void lyd_unlink_siblings(struct lyd_node *node); @@ -310,6 +315,7 @@ LY_ERR lyd_print_all(struct ly_out *, const struct lyd_node *, LYD_FORMAT, uint3 #define LYD_PARSE_OPTS_MASK ... #define LYD_PARSE_ORDERED ... #define LYD_PARSE_STRICT ... +#define LYD_PARSE_STORE_ONLY ... #define LYD_VALIDATE_NO_STATE ... #define LYD_VALIDATE_PRESENT ... @@ -614,6 +620,7 @@ struct lysp_node_list { }; struct lysc_type { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -682,7 +689,6 @@ struct lysc_ext { struct lysc_ext_instance *exts; struct lyplg_ext *plugin; struct lys_module *module; - uint32_t refcount; uint16_t flags; }; @@ -703,11 +709,10 @@ typedef enum { LYD_PATH_STD_NO_LAST_PRED } LYD_PATH_TYPE; -LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_node **); char* lyd_path(const struct lyd_node *, LYD_PATH_TYPE, char *, size_t); LY_ERR lyd_new_inner(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **); -LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **, ...); -LY_ERR lyd_new_list2(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, uint32_t, struct lyd_node **node, ...); struct lyd_node_inner { union { @@ -821,6 +826,7 @@ struct lysp_restr { }; struct lysc_type_num { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -829,6 +835,7 @@ struct lysc_type_num { }; struct lysc_type_dec { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -838,6 +845,7 @@ struct lysc_type_dec { }; struct lysc_type_str { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -859,6 +867,7 @@ struct lysc_type_bitenum_item { }; struct lysc_type_enum { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -867,6 +876,7 @@ struct lysc_type_enum { }; struct lysc_type_bits { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -875,18 +885,19 @@ struct lysc_type_bits { }; struct lysc_type_leafref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; uint32_t refcount; struct lyxp_expr *path; struct lysc_prefix *prefixes; - const struct lys_module *cur_mod; struct lysc_type *realtype; uint8_t require_instance; }; struct lysc_type_identityref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -895,6 +906,7 @@ struct lysc_type_identityref { }; struct lysc_type_instanceid { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -903,6 +915,7 @@ struct lysc_type_instanceid { }; struct lysc_type_union { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -911,6 +924,7 @@ struct lysc_type_union { }; struct lysc_type_bin { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -1053,7 +1067,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); -LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); +LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **); struct ly_opaq_name { const char *name; diff --git a/cffi/source.c b/cffi/source.c index 2682dd88..87a2e3a4 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -6,9 +6,9 @@ #include #include -#if (LY_VERSION_MAJOR != 2) -#error "This version of libyang bindings only works with libyang 2.x" +#if (LY_VERSION_MAJOR != 3) +#error "This version of libyang bindings only works with libyang 3.x" #endif -#if (LY_VERSION_MINOR < 37) -#error "Need at least libyang 2.37" +#if (LY_VERSION_MINOR < 0) +#error "Need at least libyang 3.0" #endif diff --git a/libyang/context.py b/libyang/context.py index eefc4e05..fda67d48 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -11,8 +11,8 @@ DNode, data_format, data_type, + new_path_flags, parser_flags, - path_flags, validation_flags, ) from .schema import Module, SNode, schema_in_format @@ -117,8 +117,12 @@ def error(self, msg: str, *args) -> LibyangError: while err: if err.msg: msg += ": %s" % c2str(err.msg) - if err.path: - msg += ": %s" % c2str(err.path) + if err.data_path: + msg += ": Data path: %s" % c2str(err.data_path) + if err.schema_path: + msg += ": Schema path: %s" % c2str(err.schema_path) + if err.line != 0: + msg += " (line %u)" % err.line err = err.next lib.ly_err_clean(self.cdata, ffi.NULL) @@ -234,7 +238,7 @@ def create_data_path( parent: Optional[DNode] = None, value: Any = None, update: bool = True, - no_parent_ret: bool = True, + store_only: bool = False, rpc_output: bool = False, force_return_value: bool = True, ) -> Optional[DNode]: @@ -245,9 +249,7 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = path_flags( - update=update, no_parent_ret=no_parent_ret, rpc_output=rpc_output - ) + flags = new_path_flags(update=update, store_only=store_only, output=rpc_output) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, @@ -259,7 +261,8 @@ def create_data_path( ) dnode = dnode[0] if ret != lib.LY_SUCCESS: - if lib.ly_vecode(self.cdata) != lib.LYVE_SUCCESS: + err = lib.ly_err_last(self.cdata) + if err != ffi.NULL and err.vecode != lib.LYVE_SUCCESS: raise self.error("cannot create data path: %s", path) lib.ly_err_clean(self.cdata, ffi.NULL) if not dnode and not force_return_value: diff --git a/libyang/data.py b/libyang/data.py index f0caf240..2db0261b 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -77,14 +77,84 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- -def path_flags( - update: bool = False, rpc_output: bool = False, no_parent_ret: bool = False +def new_val_flags( + output: bool = False, + store_only: bool = False, + bin_value: bool = False, + canon_value: bool = False, ) -> int: flags = 0 + if output: + flags |= lib.LYD_NEW_VAL_OUTPUT + if store_only: + flags |= lib.LYD_NEW_VAL_STORE_ONLY + if bin_value: + flags |= lib.LYD_NEW_VAL_BIN + if canon_value: + flags |= lib.LYD_NEW_VAL_CANON + return flags + + +# ------------------------------------------------------------------------------------- +def new_meta_flags( + output: bool = False, + store_only: bool = False, + bin_value: bool = False, + canon_value: bool = False, + clear_dflt: bool = False, +) -> int: + flags = new_val_flags( + output=output, + store_only=store_only, + bin_value=bin_value, + canon_value=canon_value, + ) + if clear_dflt: + flags |= lib.LYD_NEW_META_CLEAR_DFLT + return flags + + +# ------------------------------------------------------------------------------------- +def new_path_flags( + output: bool = False, + store_only: bool = False, + bin_value: bool = False, + canon_value: bool = False, + update: bool = False, + opaq: bool = False, + with_opaq: bool = False, +) -> int: + flags = new_val_flags( + output=output, + store_only=store_only, + bin_value=bin_value, + canon_value=canon_value, + ) if update: flags |= lib.LYD_NEW_PATH_UPDATE - if rpc_output: - flags |= lib.LYD_NEW_PATH_OUTPUT + if opaq: + flags |= lib.LYD_NEW_PATH_OPAQ + if with_opaq: + flags |= lib.LYD_NEW_PATH_WITH_OPAQ + return flags + + +# ------------------------------------------------------------------------------------- +def new_any_flags( + output: bool = False, + store_only: bool = False, + bin_value: bool = False, + canon_value: bool = False, + use_value: bool = False, +) -> int: + flags = new_val_flags( + output=output, + store_only=store_only, + bin_value=bin_value, + canon_value=canon_value, + ) + if use_value: + flags |= lib.LYD_NEW_ANY_USE_VALUE return flags @@ -96,6 +166,7 @@ def parser_flags( opaq: bool = False, ordered: bool = False, strict: bool = False, + store_only: bool = False, ) -> int: flags = 0 if lyb_mod_update: @@ -110,6 +181,8 @@ def parser_flags( flags |= lib.LYD_PARSE_ORDERED if strict: flags |= lib.LYD_PARSE_STRICT + if store_only: + flags |= lib.LYD_PARSE_STORE_ONLY return flags @@ -298,14 +371,17 @@ def meta_free(self, name): break item = item.next - def new_meta(self, name: str, value: str, clear_dflt: bool = False): + def new_meta( + self, name: str, value: str, clear_dflt: bool = False, store_only: bool = False + ): + flags = new_meta_flags(store_only=store_only, clear_dflt=clear_dflt) ret = lib.lyd_new_meta( ffi.NULL, self.cdata, ffi.NULL, str2c(name), str2c(value), - clear_dflt, + flags, ffi.NULL, ) if ret != lib.LY_SUCCESS: @@ -365,18 +441,16 @@ def new_path( opt_opaq: bool = False, opt_bin_value: bool = False, opt_canon_value: bool = False, + opt_store_only: bool = False, ): - opt = 0 - if opt_update: - opt |= lib.LYD_NEW_PATH_UPDATE - if opt_output: - opt |= lib.LYD_NEW_PATH_OUTPUT - if opt_opaq: - opt |= lib.LYD_NEW_PATH_OPAQ - if opt_bin_value: - opt |= lib.LYD_NEW_PATH_BIN_VALUE - if opt_canon_value: - opt |= lib.LYD_NEW_PATH_CANON_VALUE + opt = new_path_flags( + update=opt_update, + store_only=opt_store_only, + bin_value=opt_bin_value, + canon_value=opt_canon_value, + output=opt_output, + opaq=opt_opaq, + ) ret = lib.lyd_new_path( self.cdata, ffi.NULL, str2c(path), str2c(value), opt, ffi.NULL @@ -1005,10 +1079,14 @@ def _get_path(cdata) -> str: @DNode.register(SNode.CONTAINER) class DContainer(DNode): def create_path( - self, path: str, value: Any = None, rpc_output: bool = False + self, + path: str, + value: Any = None, + rpc_output: bool = False, + store_only: bool = False, ) -> Optional[DNode]: return self.context.create_data_path( - path, parent=self, value=value, rpc_output=rpc_output + path, parent=self, value=value, rpc_output=rpc_output, store_only=store_only ) def children(self, no_keys=False) -> Iterator[DNode]: @@ -1132,6 +1210,7 @@ def dict_to_dnode( rpc: bool = False, rpcreply: bool = False, notification: bool = False, + store_only: bool = False, ) -> Optional[DNode]: """ Convert a python dictionary to a DNode object given a YANG module object. The return @@ -1158,6 +1237,8 @@ def dict_to_dnode( Data represents RPC or action output parameters. :arg notification: Data represents notification parameters. + :arg store_only: + Data are being stored regardless of type validation (length, range, pattern, etc.) """ if not dic: return None @@ -1179,10 +1260,15 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False): value = str(value) n = ffi.new("struct lyd_node **") + flags = new_val_flags(output=in_rpc_output, store_only=store_only) ret = lib.lyd_new_term( - _parent, module.cdata, str2c(name), str2c(value), in_rpc_output, n + _parent, + module.cdata, + str2c(name), + str2c(value), + flags, + n, ) - if ret != lib.LY_SUCCESS: if _parent: parent_path = repr(DNode.new(module.context, _parent).path()) @@ -1211,11 +1297,12 @@ def _create_container(_parent, module, name, in_rpc_output=False): def _create_list(_parent, module, name, key_values, in_rpc_output=False): n = ffi.new("struct lyd_node **") + flags = new_val_flags(output=in_rpc_output, store_only=store_only) ret = lib.lyd_new_list( _parent, module.cdata, str2c(name), - in_rpc_output, + flags, n, *[str2c(str(i)) for i in key_values], ) diff --git a/libyang/log.py b/libyang/log.py index 2b241157..b033ccaa 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -20,13 +20,18 @@ @ffi.def_extern(name="lypy_log_cb") -def libyang_c_logging_callback(level, msg, path): +def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] - if path: - fmt = "%s: %s" - args.append(c2str(path)) - else: - fmt = "%s" + fmt = "%s" + if data_path: + fmt += ": %s" + args.append(c2str(data_path)) + if schema_path: + fmt += ": %s" + args.append(c2str(schema_path)) + if line != 0: + fmt += " line %u" + args.append(str(line)) LOG.log(LOG_LEVELS.get(level, logging.NOTSET), fmt, *args) @@ -51,10 +56,10 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non break if enable_py_logger: lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE) - lib.ly_set_log_clb(lib.lypy_log_cb, True) + lib.ly_set_log_clb(lib.lypy_log_cb) else: lib.ly_log_options(lib.LY_LOSTORE) - lib.ly_set_log_clb(ffi.NULL, False) + lib.ly_set_log_clb(ffi.NULL) configure_logging(False, logging.ERROR) diff --git a/tests/test_data.py b/tests/test_data.py index 10d9045f..a40056c0 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -132,17 +132,17 @@ def test_data_parse_config_json_without_yang_lib(self): "path": "/CESNET/libyang-python", "enabled": false }, + { + "proto": "http", + "host": "barfoo.com", + "path": "/barfoo/index.html" + }, { "proto": "http", "host": "foobar.com", "port": 8080, "path": "/index.html", "enabled": true - }, - { - "proto": "http", - "host": "barfoo.com", - "path": "/barfoo/index.html" } ], "number": [ @@ -282,7 +282,9 @@ def test_data_parse_config_xml_multi_error(self): self.assertEqual( str(cm.exception), 'failed to parse data tree: Invalid boolean value "abcd".: ' - 'List instance is missing its key "host".', + "Data path: /yolo-system:conf/url[proto='https']/enabled (line 6): " + 'List instance is missing its key "host".: ' + "Data path: /yolo-system:conf/url[proto='https'] (line 7)", ) XML_STATE = """ @@ -808,7 +810,7 @@ def test_data_to_dict_keyless_list(self): foobar.com false - + ftp github.com /CESNET/libyang-python