Feature/control append (#48)

* (wip) initial attempt to parse tags

* get update/replace/append merge semantics right

* add compositing executable

* add extensible tag processing

* clean up logger behavior and switch test variable

* Update config_utilities/include/config_utilities/internal/yaml_utils.h

Co-authored-by: Lukas Schmid <[email protected]>

* Update config_utilities/src/tag_processors.cpp

Co-authored-by: Lukas Schmid <[email protected]>

* better docstring

* revert reference removal

* add help message and parsing

* document tag behavior

* don't use special help shortopt

* syntax coloring

* touch up grammar

---------

Co-authored-by: Lukas Schmid <[email protected]>

Author: nathanhhughes

config_utilities/CMakeLists.txt

@@ -39,6 +39,7 @@ add_library(
   src/path.cpp
   src/settings.cpp
   src/string_utils.cpp
+  src/tag_processors.cpp
   src/validation.cpp
   src/visitor.cpp
   src/yaml_parser.cpp
@@ -54,6 +55,9 @@ target_compile_options(${PROJECT_NAME} PRIVATE -Wall)
 set_property(TARGET ${PROJECT_NAME} PROPERTY POSITION_INDEPENDENT_CODE 1)
 add_library(config_utilities::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
 
+add_executable(composite-configs app/composite_configs.cpp)
+target_link_libraries(composite-configs ${PROJECT_NAME})
+
 if(ENABLE_roscpp)
   target_link_libraries(${PROJECT_NAME} PUBLIC ${roscpp_LIBRARIES})
   target_include_directories(${PROJECT_NAME} PUBLIC ${roscpp_INCLUDE_DIRS})

config_utilities/app/composite_configs.cpp

@@ -0,0 +1,51 @@
+#include <iostream>
+
+#include <config_utilities/parsing/commandline.h>
+
+const std::string help_msg =
+    R"""(Usage: composite-config [--config-utilities-yaml YAML_TOKEN ...]... [--config-utilities-file FILEPATH[@NAMESPACE]]...
+
+Merges the input YAML values from left to right and outputs the resulting composite YAML to stdout.
+Invalid YAML or missing files get dropped during compositing.
+
+Options:
+  -h/--help: Show this message.
+  --config-utilities-yaml: Takes an arbitrary set of tokens that form a valid YAML string.
+                           Spaces are not required to be escaped, so `--config-utilities foo: value`
+                           is parsed the same as `--config-utilities 'foo: value'`. Can be specified
+                           multiple times.
+  --config-utilities-file: Takes a filepath to YAML to load and composite. The YAML can optionally
+                           be namespaced by `@NAMESPACE` where 'FILE@a/b' maps to
+                           '{a: {b: FILE_CONTENTS}}'. Can be specified multiple times.
+
+Example:
+> echo "{a: 42, bar: hello}" > /tmp/test_in.yaml
+> composite-configs --config-utilities-yaml "{foo: {bar: value, b: -1.0}}" --config-utilities-file /tmp/test_in.yaml@foo
+{foo: {bar: hello, b: -1.0, a: 42}}
+
+See https://github.com/MIT-SPARK/config_utilities/blob/main/docs/Parsing.md#parse-from-the-command-line
+for more information.
+)""";
+
+int main(int argc, char* argv[]) {
+  config::internal::ParserInfo info;
+  auto result = config::internal::loadFromArguments(argc, argv, false, &info);
+  if (info.help_present) {
+    std::cerr << help_msg << std::endl;
+    return 1;
+  }
+
+  switch (result.Type()) {
+    case YAML::NodeType::Null:
+    case YAML::NodeType::Undefined:
+    default:
+      break;
+    case YAML::NodeType::Scalar:
+    case YAML::NodeType::Sequence:
+    case YAML::NodeType::Map:
+      std::cout << result << std::endl;
+      break;
+  }
+
+  return 0;
+}

config_utilities/cmake/HandleInstall.cmake

@@ -1,9 +1,10 @@
 include(GNUInstallDirs)
 install(
-  TARGETS ${PROJECT_NAME}
+  TARGETS ${PROJECT_NAME} composite-configs
   EXPORT config_utilities-targets
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(
   EXPORT config_utilities-targets

config_utilities/include/config_utilities/internal/yaml_utils.h

@@ -42,12 +42,30 @@
 
 namespace config::internal {
 
+enum class MergeMode {
+  //! @brief Combine the two trees, recursing into matching sequence entries
+  UPDATE,
+  //! @brief Combine the two trees, appending right sequences into the left
+  APPEND,
+  //! @brief Combine the two trees, replacing left sequences with the right
+  REPLACE
+};
+
 /**
- * @brief Merges node b into a, overwriting values previously defined in a if they can not be
- * merged. Modifies node a, whereas b is const. Sequences can optionally be appended together at the same level of the
- * YAML tree.
+ * @brief Merges node b into a with conflicting keys handled by choice of mode
+ *
+ * Recurses through the YAML "tree" of b, adding all non-conflicting nodes to a. Conflicting nodes (i.e. map keys or
+ * shared indices in sequences that already exist in a) are handled according to the mode selection. For `REPLACE`, any
+ * conflicting node stops the recursion, and the conflicting node is replaced by the value in b. For 'APPEND', any
+ * conflicting sequence node will stop the recursion and cause the entire contents of the node in b to be append to the
+ * node in a. For 'UPDATE', any conflicting map or sequence node recursively calls `mergeYamlNodes` with the children of
+ * the conflicting nodes as the new roots.
+ *
+ * @param a Node to merge into ("left" node and will be changed).
+ * @param b Node to merge from ("right" node and remains constant).
+ * @param mode Mode to use when merging
  */
-void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences = false);
+void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, MergeMode mode = MergeMode::UPDATE);
 
 /**
  * @brief Get a pointer to the final node of the specified namespace if it exists, where each map in the yaml is

config_utilities/include/config_utilities/parsing/commandline.h

@@ -45,12 +45,18 @@
 namespace config {
 namespace internal {
 
+struct ParserInfo {
+  bool help_present = false;
+};
+
 /**
  * @brief Parse and collate YAML node from arguments, optionally removing arguments
  * @param argc Number of command line arguments
  * @param argv Command line argument strings
+ * @param remove_args Remove any recognized arguments from argc and argv
+ * @param parser_info Optional information for parser used by composite executable
  */
-YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args);
+YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args, ParserInfo* parser_info = nullptr);
 
 /**
  * @brief Parse and collate YAML node from arguments

config_utilities/include/config_utilities/tag_processors.h

@@ -0,0 +1,94 @@
+/** -----------------------------------------------------------------------------
+ * Copyright (c) 2023 Massachusetts Institute of Technology.
+ * All Rights Reserved.
+ *
+ * AUTHORS:     Lukas Schmid <[email protected]>, Nathan Hughes <[email protected]>
+ * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology
+ * YEAR:        2023
+ * LICENSE:     BSD 3-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * -------------------------------------------------------------------------- */
+
+#pragma once
+
+#include <yaml-cpp/yaml.h>
+
+namespace config {
+
+struct TagProcessor {
+  virtual ~TagProcessor() = default;
+
+  /**
+   * @brief Attempt to replace node with corresponding tag
+   * @param[in] node Node to perform substitution on
+   */
+  virtual void processNode(YAML::Node node) const = 0;
+};
+
+class RegisteredTags {
+ public:
+  template <typename T>
+  struct Registration {
+    explicit Registration(const std::string& tag);
+  };
+
+  ~RegisteredTags() = default;
+
+  static const TagProcessor* getEntry(const std::string& tag);
+
+ private:
+  template <typename T>
+  friend struct Registration;
+
+  static RegisteredTags& instance();
+
+  static void addEntry(const std::string& tag, std::unique_ptr<TagProcessor>&& proc);
+
+  RegisteredTags();
+  static std::unique_ptr<RegisteredTags> s_instance_;
+  std::map<std::string, std::unique_ptr<TagProcessor>> entries_;
+};
+
+template <typename T>
+RegisteredTags::Registration<T>::Registration(const std::string& tag) {
+  RegisteredTags::addEntry(tag, std::make_unique<T>());
+}
+
+/**
+ * @brief Attempts to replace the node `!env VARNAME` with the value of VARNAME from the environment
+ */
+struct EnvTag : public TagProcessor {
+  void processNode(YAML::Node node) const override;
+};
+
+/**
+ * @brief Iterate through the node, resolving tags
+ * @param[in] node Node to resolve tags for
+ */
+void resolveTags(YAML::Node node);
+
+}  // namespace config

config_utilities/src/commandline.cpp

@@ -36,11 +36,12 @@
 #include "config_utilities/parsing/commandline.h"
 
 #include <filesystem>
-#include <sstream>
 #include <regex>
+#include <sstream>
 
 #include "config_utilities/internal/logger.h"
 #include "config_utilities/internal/yaml_utils.h"
+#include "config_utilities/tag_processors.h"
 
 namespace config::internal {
 
@@ -60,6 +61,8 @@ struct CliParser {
     bool is_file;
   };
 
+  ParserInfo info;
+
   static constexpr auto FILE_OPT = "--config-utilities-file";
   static constexpr auto YAML_OPT = "--config-utilities-yaml";
 
@@ -143,11 +146,25 @@ void removeSpan(int& argc, char* argv[], const Span& span) {
 CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) {
   std::vector<Span> spans;
 
+  bool found_separator = false;
   int i = 0;
   while (i < argc) {
     const std::string curr_opt(argv[i]);
     std::string error;
     std::optional<Span> curr_span;
+    if ((curr_opt == "-h" || curr_opt == "--help") && !found_separator) {
+      info.help_present = true;
+      spans.emplace_back(Span{i, 0, curr_opt});
+      ++i;
+      continue;
+    }
+
+    if (curr_opt == "--" && !found_separator) {
+      found_separator = true;
+      spans.emplace_back(Span{i, 0, curr_opt});
+      ++i;
+    }
+
     if (curr_opt == FILE_OPT || curr_opt == YAML_OPT) {
       curr_span = getSpan(argc, argv, i, curr_opt == YAML_OPT, error);
     }
@@ -168,6 +185,10 @@ CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) {
   }
 
   for (const auto& span : spans) {
+    if (span.key != FILE_OPT && span.key != YAML_OPT) {
+      continue; // skip any spans for single options
+    }
+
     entries.push_back(Entry{span.extractTokens(argc, argv), span.key == FILE_OPT});
   }
 
@@ -229,8 +250,14 @@ YAML::Node nodeFromLiteralEntry(const CliParser::Entry& entry) {
   return node;
 }
 
-YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) {
+YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args, ParserInfo* info) {
   const auto parser = CliParser().parse(argc, argv, remove_args);
+  if (info) {
+    *info = parser.info;
+    if (info->help_present) {
+      return YAML::Node();
+    }
+  }
 
   YAML::Node node;
   for (const auto& entry : parser.entries) {
@@ -242,9 +269,10 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) {
     }
 
     // no-op for invalid parsed node
-    internal::mergeYamlNodes(node, parsed_node, true);
+    internal::mergeYamlNodes(node, parsed_node, MergeMode::APPEND);
   }
 
+  resolveTags(node);
   return node;
 }
 

config_utilities/src/config_context.cpp

@@ -49,7 +49,7 @@ void Context::update(const YAML::Node& other, const std::string& ns) {
   auto node = YAML::Clone(other);
   moveDownNamespace(node, ns);
   // default behavior of context is to act like the ROS1 param server and extend sequences
-  mergeYamlNodes(context.contents_, node, true);
+  mergeYamlNodes(context.contents_, node, MergeMode::APPEND);
 }
 
 void Context::clear() { instance().contents_ = YAML::Node(); }