Initial commit.

.gitignore

@@ -0,0 +1,9 @@
+certs/*
+.idea/*
+
+*.swp
+*.swo
+*.key
+*.pem
+*.req
+ca-cert

CMakeLists.txt

@@ -0,0 +1,219 @@
+cmake_minimum_required(VERSION 3.0.2)
+project(ros_kafka)
+set(CMAKE_MODULE_PATH
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmake
+    ${CMAKE_MODULE_PATH})
+
+
+## Compile as C++11, supported in ROS Kinetic and newer
+# add_compile_options(-std=c++11)
+
+## Find catkin macros and libraries
+## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz)
+## is used, also find other catkin packages
+find_package(catkin REQUIRED COMPONENTS
+  roscpp
+  std_msgs
+)
+catkin_package(CFG_EXTRAS FindLibRDKafka.cmake)
+
+## System dependencies are found with CMake's conventions
+find_package(Boost REQUIRED COMPONENTS system)
+find_package(LibRDKafka REQUIRED)
+
+
+## Uncomment this if the package has a setup.py. This macro ensures
+## modules and global scripts declared therein get installed
+## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html
+# catkin_python_setup()
+
+################################################
+## Declare ROS messages, services and actions ##
+################################################
+
+## To declare and build messages, services or actions from within this
+## package, follow these steps:
+## * Let MSG_DEP_SET be the set of packages whose message types you use in
+##   your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...).
+## * In the file package.xml:
+##   * add a build_depend tag for "message_generation"
+##   * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET
+##   * If MSG_DEP_SET isn't empty the following dependency has been pulled in
+##     but can be declared for certainty nonetheless:
+##     * add a exec_depend tag for "message_runtime"
+## * In this file (CMakeLists.txt):
+##   * add "message_generation" and every package in MSG_DEP_SET to
+##     find_package(catkin REQUIRED COMPONENTS ...)
+##   * add "message_runtime" and every package in MSG_DEP_SET to
+##     catkin_package(CATKIN_DEPENDS ...)
+##   * uncomment the add_*_files sections below as needed
+##     and list every .msg/.srv/.action file to be processed
+##   * uncomment the generate_messages entry below
+##   * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...)
+
+## Generate messages in the 'msg' folder
+# add_message_files(
+#   FILES
+#   Message1.msg
+#   Message2.msg
+# )
+
+## Generate services in the 'srv' folder
+# add_service_files(
+#   FILES
+#   Service1.srv
+#   Service2.srv
+# )
+
+## Generate actions in the 'action' folder
+# add_action_files(
+#   FILES
+#   Action1.action
+#   Action2.action
+# )
+
+## Generate added messages and services with any dependencies listed here
+# generate_messages(
+#   DEPENDENCIES
+#   sensor_msgs#   std_msgs
+# )
+
+################################################
+## Declare ROS dynamic reconfigure parameters ##
+################################################
+
+## To declare and build dynamic reconfigure parameters within this
+## package, follow these steps:
+## * In the file package.xml:
+##   * add a build_depend and a exec_depend tag for "dynamic_reconfigure"
+## * In this file (CMakeLists.txt):
+##   * add "dynamic_reconfigure" to
+##     find_package(catkin REQUIRED COMPONENTS ...)
+##   * uncomment the "generate_dynamic_reconfigure_options" section below
+##     and list every .cfg file to be processed
+
+## Generate dynamic reconfigure parameters in the 'cfg' folder
+# generate_dynamic_reconfigure_options(
+#   cfg/DynReconf1.cfg
+#   cfg/DynReconf2.cfg
+# )
+
+###################################
+## catkin specific configuration ##
+###################################
+## The catkin_package macro generates cmake config files for your package
+## Declare things to be passed to dependent projects
+## INCLUDE_DIRS: uncomment this if your package contains header files
+## LIBRARIES: libraries you create in this project that dependent projects also need
+## CATKIN_DEPENDS: catkin_packages dependent projects also need
+## DEPENDS: system dependencies of this project that dependent projects also need
+catkin_package(
+  INCLUDE_DIRS include
+#  LIBRARIES ros-kafka
+#  CATKIN_DEPENDS roscpp sensor_msgs std_msgs
+DEPENDS LibRDKafka
+)
+
+###########
+## Build ##
+###########
+
+## Specify additional locations of header files
+## Your package locations should be listed before other locations
+include_directories(
+ include
+    ${catkin_INCLUDE_DIRS}
+    ${LibRDKafka_INCLUDE_DIRS}
+)
+
+## Declare a C++ library
+# add_library(${PROJECT_NAME}
+#   src/${PROJECT_NAME}/ros-kafka.cpp
+# )
+
+## Add cmake target dependencies of the library
+## as an example, code may need to be generated before libraries
+## either from message generation or dynamic reconfigure
+# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
+
+## Declare a C++ executable
+## With catkin_make all packages are built within a single CMake context
+## The recommended prefix ensures that target names across packages don't collide
+add_executable(example_producer examples/ExampleProducer.cpp)
+add_executable(example_consumer examples/ExampleConsumer.cpp)
+
+
+
+## Rename C++ executable without prefix
+## The above recommended prefix causes long target names, the following renames the
+## target back to the shorter version for ease of user use
+## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node"
+# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "")
+
+## Add cmake target dependencies of the executable
+## same as for the library above
+add_dependencies(example_producer ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
+add_dependencies(example_consumer ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
+
+## Specify libraries to link a library or executable target against
+ target_link_libraries(example_producer
+   ${catkin_LIBRARIES}
+   ${LibRDKafka_LIBRARIES}
+ )
+
+ target_link_libraries(example_consumer
+   ${catkin_LIBRARIES}
+   ${LibRDKafka_LIBRARIES}
+ )
+
+
+#############
+## Install ##
+#############
+
+# all install targets should use catkin DESTINATION variables
+# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html
+
+## Mark executable scripts (Python etc.) for installation
+## in contrast to setup.py, you can choose the destination
+# catkin_install_python(PROGRAMS
+#   scripts/my_python_script
+#   DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+# )
+
+## Mark executables for installation
+## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html
+install(TARGETS example_producer example_consumer
+    RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+    )
+
+
+## Mark cpp header files for installation
+install(DIRECTORY include/${PROJECT_NAME}/
+    DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}
+    FILES_MATCHING PATTERN "*.h"
+    PATTERN ".svn" EXCLUDE
+    )
+# cmake find script
+install(DIRECTORY cmake
+        DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
+        PATTERN ".svn" EXCLUDE)
+## Mark other files for installation (e.g. launch and bag files, etc.)
+# install(FILES
+#   # myfile1
+#   # myfile2
+#   DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
+# )
+
+#############
+## Testing ##
+#############
+
+## Add gtest based cpp test target and link libraries
+# catkin_add_gtest(${PROJECT_NAME}-test test/test_ros-kafka.cpp)
+# if(TARGET ${PROJECT_NAME}-test)
+#   target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME})
+# endif()
+
+## Add folders to be run by python nosetests
+# catkin_add_nosetests(test)

LICENSE

@@ -0,0 +1,31 @@
+Software License Agreement (BSD License)
+
+ Copyright (c) 2023, DFKI GmbH
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following
+    disclaimer in the documentation and/or other materials provided
+    with the distribution.
+  * Neither the name of Willow Garage nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,18 @@
+***DISCLAIMER***
+
+***DO NOT USE IN PRODUCTION!!!*** 
+
+
+Dependencies (except obviously ROS):
+```console
+sudo apt install librdkafka-dev
+```
+If you want to use zstd compression one needs to compile librdkafka from source (on ubuntu 20.04).
+
+Very basic bridgin from ros<->kafka.
+Basically two template headers based on the librdkafka tutorial.
+Publishes serialized rosmsg on kafka topic assumes stamped message.
+Or consumes from kafka topic to publish on ros topic.
+See ScanProducer.cpp for the most simple example for producing to kafka
+and RosClusterConsumer reads a MarkerArray from a kafka topic.
+Custom (de-)serialization/partitioning is possible by passing a functor to the constructor.

cmake/FindLibRDKafka.cmake

@@ -0,0 +1,2 @@
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(LibRDKafka REQUIRED rdkafka++ rdkafka)

config/producer.yaml

@@ -0,0 +1,13 @@
+parameters: {
+            acks: "1",
+            message.max.bytes: "500000000",
+            queue.buffering.max.kbytes: "160000000000",
+            queue.buffering.max.messages: "10000000",
+            linger.ms: "80",
+            batch.num.messages: "16",
+            compression.type: "zstd",
+            compression.codec: "zstd",
+            compression.level: "3",
+            # 0 is infinite
+            message.timeout.ms: "0",
+       }

examples/ExampleConsumer.cpp

@@ -0,0 +1,27 @@
+#include <ros/ros.h>
+#include <ros/serialization.h>
+
+#include <ros_kafka/ros_kafka_message_consumer.h>
+#include <ros_kafka/timestamp_util.h>
+
+#include <std_msgs/String.h>
+#include <geometry_msgs/PointStamped.h>
+
+
+int main(int argc, char **argv)
+{
+  ros::init(argc, argv, "ConsumerExample");
+
+  ros::NodeHandle nh("~");
+  ros::Publisher pub = nh.advertise<std_msgs::String>("chatter", 50);
+
+  RosKafkaMessageConsumer<std_msgs::String, ros::Publisher> consumer(pub, nh, &RosKafkaSerialization::deserializeToRosMsg<std_msgs::String>);
+
+  while(ros::ok())
+  {
+      consumer.consume();
+      ros::spinOnce();
+  }
+
+  return 0;
+}

examples/ExampleProducer.cpp

@@ -0,0 +1,23 @@
+#include <ros/ros.h>
+#include <ros_kafka/ros_kafka_message_producer.h>
+#include <std_msgs/String.h>
+
+#include <ros/callback_queue.h>
+
+int main(int argc, char **argv)
+{
+  ros::init(argc, argv, "producer");
+
+  ros::NodeHandle n("~");
+  RosKafkaMessageProducer<std_msgs::String> prod(n, "chatter");
+
+  while(ros::ok())
+  {
+      prod.poll();
+      //ros::spinOnce();
+      // mimic spin behaviour according to wiki..
+      ros::getGlobalCallbackQueue()->callAvailable(ros::WallDuration(0.1));
+  }
+
+  return 0;
+}