Add TOML header and implementation

Without using it yet though

include/picongpu/plugins/openPMD/toml.cpp

@@ -0,0 +1,346 @@
+/* Copyright 2021 Franz Poeschel
+ *
+ * This file is part of PIConGPU.
+ *
+ * PIConGPU is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * PIConGPU is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with PIConGPU.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if ENABLE_OPENPMD == 1
+
+#    include "picongpu/plugins/openPMD/toml.hpp"
+
+#    include "picongpu/plugins/common/MPIHelpers.hpp"
+#    include "picongpu/plugins/misc/containsObject.hpp"
+#    include "picongpu/plugins/misc/removeSpaces.hpp"
+#    include "picongpu/plugins/misc/splitString.hpp"
+
+#    include <pmacc/communication/manager_common.hpp>
+
+// @todo Use std library function upon switching to C++17
+#    include <boost/filesystem.hpp> // boost::filesystem::exists
+
+#    include <chrono>
+#    include <sstream>
+#    include <thread> // std::this_thread::sleep_for
+#    include <utility> // std::forward
+
+#    include <mpi.h>
+#    include <toml.hpp>
+
+namespace
+{
+    using PeriodTable_t = std::vector<picongpu::toml::Periodicity>;
+
+    void mergePeriodTable(PeriodTable_t& into, PeriodTable_t&& from)
+    {
+        for(auto& periodicity : from)
+        {
+            into.push_back(std::move(periodicity));
+        }
+    }
+
+    using toml_t = toml::basic_value<toml::discard_comments>;
+
+    /*
+     * In our TOML schema, each data sink may specify a TOML table.
+     * This function parses one such table.
+     */
+    PeriodTable_t parseOnePeriodTable(toml::value::table_type const& periodTable)
+    {
+        PeriodTable_t res;
+        for(auto& pair : periodTable)
+        {
+            /*
+             * Each entry in the table may have one of the following forms:
+             * <listOfTimeSlices> = [<dataSource>*]
+             * <listOfTimeSlices> = <dataSource>
+             *
+             * Here, <listOfTimeSlices> is a string-formatted time slice in usual PIConGPU syntax, and <dataSource>
+             * is a string referring to an accepted data source.
+             */
+            std::vector<picongpu::toml::TimeSlice> timeSlices = picongpu::toml::parseTimeSlice(pair.first);
+
+            std::vector<std::string> dataSources;
+            using maybe_array_t = toml::result<toml::value::array_type const*, toml::type_error>;
+            auto dataSourcesInToml = [&pair]() -> maybe_array_t {
+                try
+                {
+                    return toml::ok(&pair.second.as_array());
+                }
+                catch(toml::type_error const& e)
+                {
+                    return toml::err(e);
+                }
+            }();
+            if(dataSourcesInToml.is_ok())
+            {
+                // 1. option: dataSources is an array:
+                for(auto& value : *dataSourcesInToml.as_ok())
+                {
+                    auto dataSource
+                        = toml::expect<std::string>(value)
+                              .or_else([](auto const&) -> toml::success<std::string> {
+                                  throw std::runtime_error("[openPMD plugin] Data sources in TOML "
+                                                           "file must be a string or a vector of strings.");
+                              })
+                              .value;
+                    dataSources.push_back(std::move(dataSource));
+                }
+            }
+            else
+            {
+                // 2. option: dataSources is no array, check if it is a simple string
+                auto dataSource = toml::expect<std::string>(pair.second)
+                                      .or_else([](auto const&) -> toml::success<std::string> {
+                                          throw std::runtime_error("[openPMD plugin] Data sources in TOML "
+                                                                   "file must be a string or a vector of strings.");
+                                      })
+                                      .value;
+                dataSources.push_back(std::move(dataSource));
+            }
+
+            for(auto& timeSlice : timeSlices)
+            {
+                res.emplace_back(picongpu::toml::Periodicity{std::move(timeSlice), dataSources});
+            }
+        }
+        return res;
+    }
+
+    void parsePluginOptions(picongpu::toml::PluginOptions& options, toml::value tomlConfig)
+    {
+        auto parseOption = [&tomlConfig](std::string& target, std::string const& key) {
+            if(not tomlConfig.contains(key))
+            {
+                return; // leave the default option
+            }
+            try
+            {
+                target = toml::find<std::string>(tomlConfig, key);
+            }
+            catch(toml::type_error const& e)
+            {
+                throw std::runtime_error(
+                    "[openPMD plugin] Global key '" + key + "' must point to a value of string type.");
+            }
+        };
+
+        parseOption(options.fileName, "file");
+        parseOption(options.fileInfix, "infix");
+        parseOption(options.fileExtension, "ext");
+        parseOption(options.jsonConfig, "backend_config");
+        parseOption(options.dataPreparationStrategy, "data_preparation_strategy");
+    }
+
+    PeriodTable_t parseTomlFile(
+        picongpu::toml::DataSources& dataSources,
+        std::string const& content,
+        std::string const& file = "unknown file")
+    {
+        auto data = [&content, &file]() {
+            std::istringstream istream(content.c_str());
+            return toml::parse(istream, file);
+        }();
+
+        parsePluginOptions(dataSources.openPMDPluginOptions, data);
+
+        if(not data.contains("sink"))
+        {
+            return {};
+        }
+        auto& sinkTable = [&data]() -> toml::value::table_type const& {
+            try
+            {
+                return toml::find(data, "sink").as_table();
+            }
+            catch(toml::type_error const& e)
+            {
+                throw std::runtime_error(
+                    "[openPMD plugin] Key 'sink' in TOML file must be a table (" + std::string(e.what()) + ")");
+            }
+        }();
+
+        PeriodTable_t period;
+        for(auto const& pair : sinkTable)
+        {
+            std::string const& sinkName = pair.first;
+            (void) sinkName; // we don't need it for now
+            toml::value const& perSink = pair.second;
+            if(not perSink.contains("period"))
+            {
+                continue;
+            }
+            auto& periodTable = [&perSink]() -> toml::value::table_type const& {
+                try
+                {
+                    return toml::find(perSink, "period").as_table();
+                }
+                catch(toml::type_error const& e)
+                {
+                    throw std::runtime_error(
+                        "[openPMD plugin] Key 'period' in TOML file must be a table (" + std::string(e.what()) + ")");
+                }
+            }();
+            mergePeriodTable(period, parseOnePeriodTable(periodTable));
+        }
+        return period;
+    }
+
+    template<typename ChronoDuration>
+    PeriodTable_t waitForAndParseTomlFile(
+        picongpu::toml::DataSources& dataSources,
+        std::string const path,
+        ChronoDuration&& sleepInterval,
+        MPI_Comm comm)
+    {
+        int rank;
+        MPI_CHECK(MPI_Comm_rank(comm, &rank));
+        {
+            char const* argsv[] = {path.c_str()};
+            picongpu::toml::writeLog("openPMD: Reading data requirements from TOML file: '%1%'", 1, argsv);
+        }
+
+        // wait for file to appear
+        if(rank == 0)
+        {
+            char const* argsv[] = {path.c_str()};
+            while(!boost::filesystem::exists(path))
+            {
+                picongpu::toml::writeLog("openPMD: Still waiting for TOML file:\n\t%1%", 1, argsv);
+                std::this_thread::sleep_for(sleepInterval);
+            }
+        }
+
+        MPI_CHECK(MPI_Barrier(comm));
+
+        picongpu::toml::writeLog("openPMD: Reading TOML file collectively");
+        std::string fileContents = picongpu::collective_file_read(path, comm);
+
+        return parseTomlFile(dataSources, fileContents, path);
+    }
+} // namespace
+
+
+namespace picongpu
+{
+    namespace toml
+    {
+        using namespace std::literals::chrono_literals;
+        constexpr std::chrono::seconds const WAIT_TIME = 5s;
+
+        std::string TimeSlice::asString() const
+        {
+            return std::to_string(start) + ':' + std::to_string(end) + ':' + std::to_string(period);
+        }
+
+        DataSources::DataSources(
+            std::string const& tomlFile,
+            std::vector<std::string> const& allowedDataSources,
+            MPI_Comm comm)
+        {
+            std::vector<Periodicity> period = waitForAndParseTomlFile(*this, tomlFile, WAIT_TIME, comm);
+            for(auto& periodicity : period)
+            {
+                auto start = periodicity.timeSlice.start;
+                for(auto const& source : periodicity.sources)
+                {
+                    if(!plugins::misc::containsObject(allowedDataSources, source))
+                    {
+                        throw std::runtime_error("[openPMD plugin]: unknown data source '" + source + "'");
+                    }
+                }
+                m_upcomingSteps[start].push_back(std::move(periodicity));
+            }
+        }
+
+        std::vector<std::string> DataSources::currentDataSources() const
+        {
+            if(m_upcomingSteps.empty())
+            {
+                return {};
+            }
+
+            std::set<std::string> result;
+            for(Periodicity const& period : m_upcomingSteps.begin()->second)
+            {
+                for(std::string const& source : period.sources)
+                {
+                    result.insert(source);
+                }
+            }
+            return {result.begin(), result.end()};
+        }
+
+        auto DataSources::currentStep() const -> SimulationStep_t
+        {
+            if(m_upcomingSteps.empty())
+            {
+                return std::numeric_limits<SimulationStep_t>::max();
+            }
+            else
+            {
+                return m_upcomingSteps.begin()->first;
+            }
+        }
+
+        DataSources& DataSources::operator++()
+        {
+            if(m_upcomingSteps.empty())
+            {
+                return *this;
+            }
+            SimulationStep_t current = currentStep();
+            for(Periodicity& period : m_upcomingSteps.begin()->second)
+            {
+                auto nextStep = current + period.timeSlice.period;
+                // time slice end parameter is inclusive
+                if(nextStep > period.timeSlice.end)
+                {
+                    // time slice is finished
+                    continue;
+                }
+                m_upcomingSteps[nextStep].push_back(std::move(period));
+            }
+            m_upcomingSteps.erase(m_upcomingSteps.begin());
+            return *this;
+        }
+
+        std::string DataSources::periods() const
+        {
+            std::vector<std::string> notConcatenated;
+            for(auto const& step : m_upcomingSteps)
+            {
+                for(auto const& periodicity : step.second)
+                {
+                    notConcatenated.push_back(periodicity.timeSlice.asString());
+                }
+            }
+            if(notConcatenated.empty())
+            {
+                return {};
+            }
+            std::stringstream stringBuilder;
+            auto iterator = notConcatenated.begin();
+            stringBuilder << *iterator;
+            for(; iterator != notConcatenated.end(); ++iterator)
+            {
+                stringBuilder << "," << *iterator;
+            }
+            return stringBuilder.str();
+        }
+    } // namespace toml
+} // namespace picongpu
+
+#endif // ENABLE_OPENPMD