[WIP] Initial draft of python interface with pybind

CMakeLists.txt

@@ -70,6 +70,10 @@ ecbuild_add_option( FEATURE FDB_REMOTE
                     DEFAULT ON
                     DESCRIPTION "Support for FDB remote access" )
 
+ecbuild_add_option( FEATURE PYBIND
+                    DEFAULT ON
+                    DESCRIPTION "Support for PyFDB generation" )
+
 find_package(UUID QUIET)
 
 find_package(DAOS QUIET)
@@ -169,3 +173,32 @@ add_subdirectory( tests )
 ecbuild_install_project( NAME fdb )
 
 ecbuild_print_summary()
+
+
+# set(PYBIND11_FINDPYTHON ON)
+# find_package(pybind11 CONFIG REQUIRED)
+#
+# pybind11_add_module(example src/fdb5/api/fdb_py.cc)
+#
+# target_link_libraries(example PRIVATE 
+#                     eckit
+#                     eccodes
+#                     metkit
+#                     fdb5
+#                     pybind11::module 
+#                     pybind11::lto 
+#                     pybind11::windows_extras)
+#
+# pybind11_extension(example)
+# if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
+#     # Strip unnecessary sections of the binary on Linux/macOS
+#     pybind11_strip(example)
+# endif()
+#
+# set_target_properties(example PROPERTIES CXX_VISIBILITY_PRESET "hidden"
+#                                          CUDA_VISIBILITY_PRESET "hidden")
+#
+#
+# install(TARGETS example DESTINATION .)
+#
+#

pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = ["scikit-build-core", "pybind11"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "example"
+version = "0.1.0"
+

src/fdb5/CMakeLists.txt

@@ -540,3 +540,11 @@ if ( HAVE_FDB_BUILD_TOOLS )
     target_sources( fdb-lock PRIVATE tools/FDBLock.cc tools/FDBLock.h )
     target_sources( fdb-unlock PRIVATE tools/FDBLock.cc tools/FDBLock.h )
 endif()
+
+
+## Exports headers for pyfdb
+if ( HAVE_PYBIND ) 
+    install(FILES api/fdb_py.cc DESTINATION include/fdb5/api)
+    target_include_directories(fdb5 PRIVATE ${INSTALL_INCLUDE_DIR})
+endif()
+

src/fdb5/api/fdb_py.cc

@@ -0,0 +1,90 @@
+#include <pybind11/pybind11.h>
+#include <pybind11/iostream.h>
+#include <pybind11/stl.h>
+#include <cstddef>
+
+#include "eckit/config/Configuration.h"
+#include "eckit/eckit_version.h"
+#include "eckit/filesystem/PathName.h"
+#include "eckit/io/DataHandle.h"
+#include "eckit/io/TransferWatcher.h"
+#include "eckit/message/Message.h"
+#include "fdb5/api/FDB.h"
+#include "fdb5/api/FDBFactory.h"
+#include "fdb5/api/LocalFDB.h"
+#include "fdb5/api/fdb_c.h"
+#include "fdb5/api/helpers/FDBToolRequest.h"
+#include "fdb5/config/Config.h"
+#include "fdb5/database/Inspector.h"
+#include "fdb5/database/Archiver.h"
+#include "fdb5/database/Key.h"
+#include "metkit/mars/Parameter.h"
+
+namespace py = pybind11;
+
+
+PYBIND11_MODULE(_core, module) {
+    module.doc() = "pyfdb c++ api"; // optional module docstring
+
+    py::class_<eckit::PathName>(module, "PathName")
+       .def(py::init<const std::string&, bool>())
+       .def("asString", &eckit::PathName::asString);
+
+    py::class_<eckit::LocalConfiguration>(module, "LocalConfiguration")
+        .def(py::init<const eckit::Configuration&, const eckit::PathName&>());
+        .def(py::init([](const std::unordered_map<std::string, std::string> key_value_map){
+                    auto result_map = eckit::LocalConfiguration();
+                    for()
+                    }))
+
+    py::class_<fdb5::Config>(module, "Config")
+        .def(py::init<>())
+        .def(py::init<const eckit::Configuration&, const eckit::Configuration&>());
+
+    py::class_<metkit::mars::MarsRequest>(module, "MarsRequest")
+        .def(py::init<>())
+        .def(py::init<const std::string&>())
+        .def(py::init<const std::string&, const std::map<std::string, std::string>&>())
+        .def("asString", &metkit::mars::MarsRequest::asString);
+
+    py::class_<fdb5::FDBToolRequest>(module, "FDBToolRequest")
+        .def(py::init<const metkit::mars::MarsRequest&, bool, const std::vector<std::string>&>());
+
+    py::class_<fdb5::ListElement>(module, "ListElement")
+        .def(py::init<>())
+        .def("__str__", [](fdb5::ListElement& el){
+                std::stringstream buffer;
+                buffer << el;
+                return buffer.str();
+                });
+
+    py::class_<fdb5::APIIterator<fdb5::ListElement>>(module, "APIIterator");
+    py::class_<fdb5::ListIterator, fdb5::APIIterator<fdb5::ListElement>>(module, "ListIterator")
+        .def("__iter__", [](fdb5::ListIterator &it) -> fdb5::ListIterator& { return it; })
+        .def("__next__", &fdb5::ListIterator::next)
+        .def("next", &fdb5::ListIterator::next);
+
+    py::class_<eckit::DataHandle>(module, "DataHandle")
+        .def("open", &eckit::DataHandle::openForRead)
+        .def("close", &eckit::DataHandle::close)
+        .def("skip", &eckit::DataHandle::skip)
+        .def("seek", &eckit::DataHandle::seek)
+        .def("tell", &eckit::DataHandle::position)
+        .def("read", &eckit::DataHandle::read)
+        .def("save_into", [](eckit::DataHandle& dh, const eckit::PathName& path){
+                    dh.saveInto(path);
+                });
+
+    py::class_<fdb5::FDB>(module, "FDB")
+        .def(py::init([](const fdb5::Config& config){
+                    fdb_initialise();
+                    return fdb5::FDB(config);
+                    }))
+        .def("list", &fdb5::FDB::list)
+        .def("archive", pybind11::overload_cast<const void*, std::size_t>(&fdb5::FDB::archive), "Archive data given as a byte stream with the corresponding length")
+        .def("archive", pybind11::overload_cast<const metkit::mars::MarsRequest&, eckit::DataHandle&>(&fdb5::FDB::archive), "Archive a MarsRequest and its corresponding DataHandle&")
+        .def("flush", &fdb5::FDB::flush)
+        .def("retrieve", &fdb5::FDB::retrieve)
+        .doc() = "FDB instance.";
+
+}

src/fdb5/tools/FDBLock.cc

@@ -12,6 +12,7 @@
 
 #include "eckit/log/Log.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"

src/fdb5/tools/FDBTool.cc

@@ -9,14 +9,11 @@
  */
 
 #include "eckit/option/CmdArgs.h"
-#include "eckit/types/Date.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/LibFdb5.h"
-#include "fdb5/rules/Schema.h"
 #include "fdb5/tools/FDBTool.h"
 
-using eckit::Log;
-
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -34,7 +31,9 @@ static void usage(const std::string& tool) {
 }
 
 void FDBTool::run() {
-    options_.push_back(new eckit::option::SimpleOption<std::string>("config", "FDB configuration filename"));
+    if(needsConfig_) {
+        options_.push_back(new eckit::option::SimpleOption<std::string>("config", "FDB configuration filename"));
+    }
 
     eckit::option::CmdArgs args(&fdb5::usage, options_, numberOfPositionalArguments(),
                                 minimumPositionalArguments());

src/fdb5/tools/FDBTool.h

@@ -19,10 +19,11 @@
 #include <vector>
 
 #include "eckit/runtime/Tool.h"
-#include "eckit/filesystem/PathName.h"
+#include "eckit/config/Configuration.h"
+#include "eckit/config/LocalConfiguration.h"
+#include "eckit/exception/Exceptions.h"
 
-#include "fdb5/database/DB.h"
-#include "eckit/option/SimpleOption.h"
+#include "fdb5/config/Config.h"
 
 namespace eckit {
     namespace option {
@@ -55,6 +56,8 @@ class FDBTool : public eckit::Tool {
 protected: // members
 
     std::vector<eckit::option::Option *> options_;
+    /// Set this to false in tool subclass if your tool does not require access to 'config.yaml'
+    bool needsConfig_{true};
 
 protected: // methods
 

src/fdb5/tools/fdb-copy.cc

@@ -11,8 +11,8 @@
 #include <fstream>
 #include <memory>
 
-#include "eckit/io/FileHandle.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 #include "eckit/filesystem/PathName.h"
 
 #include "metkit/mars/MarsRequest.h"

src/fdb5/tools/fdb-dump-index.cc

@@ -11,23 +11,19 @@
 #include "eckit/option/CmdArgs.h"
 
 #include "fdb5/toc/TocHandler.h"
-#include "fdb5/toc/TocIndex.h"
 #include "fdb5/tools/FDBTool.h"
 
-#include "fdb5/toc/BTreeIndex.h"
-#include "fdb5/database/Index.h"
-
 #include <string>
 
 using namespace eckit;
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class FDBDumpToc : public fdb5::FDBTool {
+class FDBDumpIndex : public fdb5::FDBTool {
 
   public: // methods
 
-    FDBDumpToc(int argc, char **argv) :
+    FDBDumpIndex(int argc, char **argv) :
         fdb5::FDBTool(argc, argv) {}
 
   private: // methods
@@ -36,14 +32,14 @@ class FDBDumpToc : public fdb5::FDBTool {
     virtual void execute(const option::CmdArgs& args);
 };
 
-void FDBDumpToc::usage(const std::string &tool) const {
+void FDBDumpIndex::usage(const std::string &tool) const {
     Log::info() << std::endl
                 << "Usage: " << tool << " [path1] [path2] ..." << std::endl;
     fdb5::FDBTool::usage(tool);
 }
 
 
-void FDBDumpToc::execute(const option::CmdArgs& args) {
+void FDBDumpIndex::execute(const option::CmdArgs& args) {
 
     // n.b. We don't just open the toc, then check if in the list of indexes, as there
     //      is no reason to think that the indexes map to the same toc (or directory).
@@ -62,7 +58,7 @@ void FDBDumpToc::execute(const option::CmdArgs& args) {
 //----------------------------------------------------------------------------------------------------------------------
 
 int main(int argc, char **argv) {
-    FDBDumpToc app(argc, argv);
+    FDBDumpIndex app(argc, argv);
     return app.start();
 }
 

src/fdb5/tools/fdb-dump-toc.cc

@@ -9,6 +9,7 @@
  */
 
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/toc/TocHandler.h"
 #include "fdb5/tools/FDBTool.h"
@@ -21,9 +22,9 @@ class FDBDumpToc : public fdb5::FDBTool {
 
   public: // methods
 
-    FDBDumpToc(int argc, char **argv) :
-        fdb5::FDBTool(argc, argv) {
-
+    FDBDumpToc(int argc, char **argv) : fdb5::FDBTool(argc, argv) {
+        // FDBDumpToc does not require to read the configuration
+        needsConfig_ = false;
         options_.push_back(new eckit::option::SimpleOption<bool>("walk", "Walk subtocs rather than show simple entries"));
     }
 

src/fdb5/tools/fdb-dump.cc

@@ -13,8 +13,6 @@
 
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Index.h"
-#include "fdb5/rules/Schema.h"
 #include "fdb5/tools/FDBVisitTool.h"
 
 using namespace eckit;

src/fdb5/tools/fdb-hammer.cc

@@ -23,7 +23,6 @@
 #include "eckit/io/EmptyHandle.h"
 #include "eckit/option/CmdArgs.h"
 #include "eckit/option/SimpleOption.h"
-#include "eckit/option/VectorOption.h"
 
 #include "fdb5/message/MessageArchiver.h"
 #include "fdb5/io/HandleGatherer.h"
@@ -338,7 +337,7 @@ void FDBHammer::executeList(const eckit::option::CmdArgs &args) {
         number_values.push_back(std::to_string(n + number - 1));
     }
     request.values("number", number_values);
-   
+
     std::vector<std::string> levelist_values;
     for (size_t l = 1; l <= nlevels; ++l) {
         levelist_values.push_back(std::to_string(l + level - 1));
@@ -357,7 +356,7 @@ void FDBHammer::executeList(const eckit::option::CmdArgs &args) {
 
     size_t count = 0;
     for (size_t step = 0; step < nsteps; ++step) {
- 
+
         request.setValue("step", step);
 
         auto listObject = fdb.list(fdb5::FDBToolRequest(request, false, minimumKeys));

src/fdb5/tools/fdb-hide.cc

@@ -8,18 +8,15 @@
  * does it submit to any jurisdiction.
  */
 
-#include "eckit/config/Resource.h"
 #include "eckit/option/CmdArgs.h"
-#include "eckit/option/VectorOption.h"
-#include "eckit/os/AutoUmask.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
 #include "fdb5/toc/TocCatalogueWriter.h"
-#include "fdb5/toc/TocCatalogueReader.h"
 #include "fdb5/toc/TocEngine.h"
 #include "fdb5/tools/FDBTool.h"
 

src/fdb5/tools/fdb-info.cc

@@ -9,15 +9,12 @@
  */
 
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/LibFdb5.h"
+#include "fdb5/tools/FDBTool.h"
 #include "fdb5/config/Config.h"
-#include "fdb5/database/DB.h"
-#include "fdb5/database/Index.h"
-#include "fdb5/tools/FDBInspect.h"
 #include "fdb5/io/LustreSettings.h"
-
-#include "fdb5/fdb5_config.h"
 #include "fdb5/fdb5_version.h"
 
 using eckit::Log;