AXL: add configuration call

doc/README.md

@@ -19,6 +19,49 @@ If no state files is needed, one may pass NULL in place of the path name.
 
 One must call AXL\_Finalize to shut down the library.
 
+# Setting options
+AXL contains several tunable options that control how files are transferred.
+These are settable using the AXL\_Config function which takes as its single
+argument a pointer to a kvtree of option-value pairs. On success a non-NULL
+pointer is returned, on failure NULL is returned. Calling AXL\_Config with a
+NULL pointer instead returns a kvtree with the current values of all settable
+configuration options. It must be freed used kvtree\_delete after use.  A
+subset of options can also be set for individual AXL transfers by passing them
+in a subtree "id" of the kvtree indexed by their integer transfer id. These
+options are included in the result of AXL\_Config as an "id" subtree. By
+default a transfer will use the values of the global configuration options
+present at the time the transfer is created. Changing options of a transfer
+after is has been dispatched results in undefined behavior.
+
+An example kvtree may look like this:
+
+    +- FILE_BUF_SIZE
+    |  +- 1000000
+    +- DEBUG
+    |  +- 0
+    +- id
+       +- 42
+       |  +- FILE_BUF_SIZE
+       |  |  +- 1048576
+       |  +- COPY_METADATA 
+       |     +- 1
+       +- 1
+          +- FILE_BUF_SIZE
+             +- 65536
+
+axl.h defines symbolic names for all parameter which should be used instead of
+the immediate strings whenever possible to avoid typos and runtime errors.
+
+The current set of configuariont options including their default value is
+(prefix Name by AXL\_KEY\_CONFIG\_ for symbolic name):
+
+Name | Type  | Default | Per transfer | Description
+-----|------:|-----:|:-:|--------------------------------------------------------
+FILE\_BUF\_SIZE | Byte count | 1048576 | Yes | Specify the number of bytes to use for internal buffers when copying files between source and destination.
+DEBUG           |    Boolean |       0 |  No | Set to 1 to have AXL print debug messages to stdout, set to 0 for no output.
+MKDIR           |    Boolean |       1 | Yes | Specifies whether the destination file system supports the creation of directories (1) or not (0).
+COPY\_METADATA  |    Boolean |       0 | Yes | Whether file metadata like timestamp and permission bits should also be copied.
+
 # Transferring files
 Regardless of the transfer type, the basic control flow of a transfer is always:
 1. AXL\_Create - allocate a new transfer object, providing its type and a name

src/axl.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
@@ -245,7 +246,7 @@ int __AXL_Init (const char* state_file)
     int rc = AXL_SUCCESS;
 
     /* TODO: set these by config file */
-    axl_file_buf_size = (size_t) 1048576;
+    axl_file_buf_size = (unsigned long) 1048576;
 
     /* initialize our debug level for filtering AXL_DBG messages */
     axl_debug = 0;
@@ -309,6 +310,236 @@ int AXL_Finalize (void)
     return rc;
 }
 
+/** Actual function to set config parameters */
+static kvtree* AXL_Config_Set(const kvtree* config)
+{
+    assert(config != NULL);
+
+    kvtree* retval = (kvtree*)(config);
+
+    static const char* known_global_options[] = {
+        AXL_KEY_CONFIG_FILE_BUF_SIZE,
+        AXL_KEY_CONFIG_DEBUG,
+        AXL_KEY_CONFIG_MKDIR,
+        AXL_KEY_CONFIG_COPY_METADATA,
+        NULL
+    };
+    static const char* known_transfer_options[] = {
+        AXL_KEY_CONFIG_FILE_BUF_SIZE,
+        AXL_KEY_CONFIG_MKDIR,
+        AXL_KEY_CONFIG_COPY_METADATA,
+        NULL
+    };
+
+    /* global options */
+    /* TODO: this could be turned into a list of structs */
+   kvtree_util_get_bytecount(config, AXL_KEY_CONFIG_FILE_BUF_SIZE,
+                             &axl_file_buf_size);
+
+    kvtree_util_get_int(config, AXL_KEY_CONFIG_DEBUG, &axl_debug);
+
+    kvtree_util_get_int(config, AXL_KEY_CONFIG_MKDIR, &axl_make_directories);
+
+    kvtree_util_get_int(config, AXL_KEY_CONFIG_COPY_METADATA,
+        &axl_copy_metadata);
+
+    /* check for local options inside an "id" subkey */
+    kvtree* ids = kvtree_get(config, "id");
+    if (ids != NULL) {
+        const kvtree_elem* elem;
+        for (elem = kvtree_elem_first(ids); elem != NULL && retval != NULL;
+             elem = kvtree_elem_next(elem))
+        {
+            const char* key = kvtree_elem_key(elem);
+            char* endptr;
+            long id = strtol(key, &endptr, 10);
+            if ((*key == '\0' || *endptr != '\0') ||
+                (id < 0 || id >= axl_kvtrees_count)) {
+                retval = NULL;
+                break;
+            }
+            kvtree* local_config = kvtree_elem_hash(elem);
+            if (local_config == NULL) {
+                retval = NULL;
+                break;
+            }
+
+            /* this should be protected by a mutex_lock to prevent issues with
+             * realloc() moving memory when growing axl_kvtrees, but no one else
+             * does ... */
+            kvtree* file_list = axl_kvtrees[id];
+
+            const char** opt;
+            for (opt = known_transfer_options; *opt != NULL; opt++) {
+                const char* val = kvtree_get_val(local_config, *opt);
+                if (val != NULL) {
+                    /* this is (annoyingly) non-atomic so could leave file_list in
+                     * a strange state if malloc() fails at the wrong time.
+                     * Using kvtree_merge with a temporary tree does not seem to be any
+                     * better though */
+                    if (kvtree_util_set_str(file_list, *opt, val) != KVTREE_SUCCESS) {
+                        retval = NULL;
+                        break;
+                    }
+                }
+            }
+
+            /* report all unknown options (typos?) */
+            /* TODO: move into a function, is used twice (well almost, differs in
+             * whether "id" is acceptable */
+            const kvtree_elem* local_elem;
+            for (local_elem = kvtree_elem_first(local_config);
+                 local_elem != NULL;
+                 local_elem = kvtree_elem_next(local_elem))
+            {
+                /* must be only one level deep, ie plain kev = value */
+                const kvtree* elem_hash = kvtree_elem_hash(local_elem);
+                assert(kvtree_size(elem_hash) == 1);
+
+                const kvtree* kvtree_first_elem_hash =
+                  kvtree_elem_hash(kvtree_elem_first(elem_hash));
+                assert(kvtree_size(kvtree_first_elem_hash) == 0);
+
+                /* check against known options */
+                const char** opt;
+                int found = 0;
+                for (opt = known_transfer_options; *opt != NULL; opt++) {
+                    if (strcmp(*opt, kvtree_elem_key(local_elem)) == 0) {
+                        found = 1;
+                        break;
+                    }
+                }
+                if (! found) {
+                    AXL_ERR("Unknown configuration parameter '%s' with value '%s'",
+                      kvtree_elem_key(local_elem),
+                      kvtree_elem_key(kvtree_elem_first(kvtree_elem_hash(local_elem)))
+                    );
+                    retval = NULL;
+                    break;
+                }
+            }
+        }
+    }
+
+    /* report all unknown options (typos?) */
+    const kvtree_elem* elem;
+    for (elem = kvtree_elem_first(config);
+         elem != NULL;
+         elem = kvtree_elem_next(elem))
+    {
+        const char* key = kvtree_elem_key(elem);
+        if (strcmp("id", key) == 0)
+            continue;
+
+        /* must be only one level deep, ie plain kev = value */
+        const kvtree* elem_hash = kvtree_elem_hash(elem);
+        assert(kvtree_size(elem_hash) == 1);
+
+        const kvtree* kvtree_first_elem_hash =
+          kvtree_elem_hash(kvtree_elem_first(elem_hash));
+        assert(kvtree_size(kvtree_first_elem_hash) == 0);
+
+        /* check against known options */
+        const char** opt;
+        int found = 0;
+        for (opt = known_global_options; *opt != NULL; opt++) {
+            if (strcmp(*opt, key) == 0) {
+                found = 1;
+                break;
+            }
+        }
+        if (! found) {
+            AXL_ERR("Unknown configuration parameter '%s' with value '%s'",
+              key, kvtree_elem_key(kvtree_elem_first(kvtree_elem_hash(elem)))
+            );
+            retval = NULL;
+        }
+    }
+
+    return retval;
+}
+
+/** Actual function to get config parameters */
+static kvtree* AXL_Config_Get()
+{
+    kvtree* config = kvtree_new();
+    assert(config != NULL);
+
+    static const char* known_options[] = {
+        AXL_KEY_CONFIG_FILE_BUF_SIZE,
+        AXL_KEY_CONFIG_MKDIR,
+        AXL_KEY_CONFIG_COPY_METADATA,
+        NULL
+    };
+
+    int success = 1; /* all values could be set? */
+
+    /* global options */
+    success &= kvtree_util_set_bytecount(config,
+                                         AXL_KEY_CONFIG_FILE_BUF_SIZE,
+                                         axl_file_buf_size)
+                 == KVTREE_SUCCESS;
+    success &= kvtree_util_set_int(config, AXL_KEY_CONFIG_DEBUG, axl_debug)
+                 == KVTREE_SUCCESS;
+    success &= kvtree_util_set_int(config, AXL_KEY_CONFIG_MKDIR,
+                                   axl_make_directories) == KVTREE_SUCCESS;
+    success &= kvtree_util_set_int(config, AXL_KEY_CONFIG_COPY_METADATA,
+                                   axl_copy_metadata) == KVTREE_SUCCESS;
+
+    /* per transfer options */
+    /* this should be protected by a mutex_lock to prevent issues with
+     * realloc() moving memory when growing axl_kvtrees, but no one else
+     * does ... */
+    int id;
+    for (id = 0; id < axl_kvtrees_count; id++) {
+        kvtree* file_list = axl_kvtrees[id];
+        /* TODO: check if it would be better to return an empty hash instead */
+        if (file_list == NULL)
+            continue;
+
+        kvtree* new = kvtree_set_kv_int(config, "id", id);
+        if (new == NULL) {
+            success = 0;
+            break;
+        }
+        /* get all known options */
+        const char** opt;
+        for (opt = known_options; *opt != NULL; opt++) {
+            const char* val = kvtree_get_val(file_list, *opt);
+            if (val != NULL) {
+                /* this is (annoyingly) non-atomic so could leave new in
+                 * a strange state if malloc() fails at the wrong time.
+                 * Using kvtree_merge with a temporary tree does not seem to be any
+                 * better though */
+                if (kvtree_util_set_str(new, *opt, val) != KVTREE_SUCCESS) {
+                    success = 0;
+                    break;
+                }
+            } else {
+                /* these options must exist, if not something is wrong */
+                success = 0;
+                break;
+            }
+        }
+    }
+    if (!success) {
+        kvtree_delete(&config);
+    }
+
+    return config;
+}
+
+/** Set a AXL config parameters */
+kvtree* AXL_Config(const kvtree* config)
+{
+    if (config != NULL) {
+        return AXL_Config_Set(config);
+    } else {
+        return AXL_Config_Get();
+    }
+}
+
+
 /* Create a transfer handle (used for 0+ files)
  * Type specifies a particular method to use
  * Name is a user/application provided string
@@ -372,6 +603,21 @@ int __AXL_Create(axl_xfer_t xtype, const char* name, const char* state_file)
         kvtree_util_set_str(file_list, AXL_KEY_UNAME, name);
         kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_SOURCE);
         kvtree_util_set_int(file_list, AXL_KEY_STATE, (int)AXL_XFER_STATE_CREATED);
+
+        /* options */
+        /* TODO: handle return code of kvtree_util_set_XXX */
+        /* TODO: handle differnces between size_t and unsigned long */
+        kvtree_util_set_bytecount(file_list,
+                                  AXL_KEY_CONFIG_FILE_BUF_SIZE,
+                                  axl_file_buf_size);
+        /* a per transfer debug value is not currently supported
+        success &= kvtree_util_set_int(file_list, AXL_KEY_CONFIG_DEBUG, axl_debug)
+                     == KVTREE_SUCCESS;
+        */
+        kvtree_util_set_int(file_list, AXL_KEY_CONFIG_MKDIR,
+                            axl_make_directories);
+        kvtree_util_set_int(file_list, AXL_KEY_CONFIG_COPY_METADATA,
+                            axl_copy_metadata);
     }
 
     /* create a structure based on transfer type */
@@ -697,8 +943,13 @@ int __AXL_Dispatch (int id, int resume)
 
     kvtree_util_set_int(file_list, AXL_KEY_STATE, (int)AXL_XFER_STATE_DISPATCHED);
 
+    int make_directories;
+    int success = kvtree_util_get_int(file_list, AXL_KEY_CONFIG_MKDIR,
+                                      &make_directories);
+    assert(success == KVTREE_SUCCESS);
+
     /* create destination directories for each file */
-    if (axl_make_directories) {
+    if (make_directories) {
         while ((elem = axl_get_next_path(id, elem, NULL, &dest))) {
             char* dest_path = strdup(dest);
             char* dest_dir = dirname(dest_path);

src/axl.h

@@ -24,6 +24,12 @@ extern "C" {
  *  \ingroup axl
  *  \brief asynchronous transfer library */
 
+/** AXL configuration options */
+#define AXL_KEY_CONFIG_FILE_BUF_SIZE "FILE_BUF_SIZE"
+#define AXL_KEY_CONFIG_DEBUG "DEBUG"
+#define AXL_KEY_CONFIG_MKDIR "MKDIR"
+#define AXL_KEY_CONFIG_COPY_METADATA "COPY_METADATA"
+
 /** Supported AXL transfer methods
  * Note that DW, BBAPI, and CPPR must be found at compile time */
 typedef enum {
@@ -82,6 +88,23 @@ int AXL_Finalize (void);
         __AXL_Create(type, name, GET_ARG0(__VA_ARGS__))
 int __AXL_Create (axl_xfer_t xtype, const char* name, const char* state_file);
 
+/**
+ * Get/set AXL configuration values.
+ *
+ * config: The new configuration.  Global variables are in top level of
+ *              the tree, and per-ID values are subtrees.  If config=NULL,
+ *              then return a kvtree with all the configuration values (globals
+ *              and all per-ID trees).
+ *
+ * Return value: If config != NULL, then return config on success.  If
+ *                      config=NULL (you're querying the config) then return
+ *                      a new kvtree on success.  Return NULL on any failures.
+ */
+typedef struct kvtree_struct kvtree;
+kvtree* AXL_Config(
+  const kvtree* config        /** [IN] - kvtree of options */
+);
+
 /** Add a file to an existing transfer handle */
 int AXL_Add (int id, const char* source, const char* destination);