Forward Error Correction

.github/workflows/build.yml

@@ -0,0 +1,27 @@
+name: build
+on: [push]
+jobs:
+  build:
+    strategy:
+      matrix:
+        build-type: [Release]
+        raptor: [ON, OFF]
+        os: [ubuntu-latest, ubuntu-22.04, ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: install dependencies
+        run: sudo apt install -y gcc ninja-build cmake libssl-dev libboost-all-dev libspdlog-dev libtinyxml2-dev libnl-3-dev pkg-config libconfig++-dev
+      - name: pre-cmake
+        run: cmake --version && mkdir ${{ github.workspace }}/build && echo `nproc` processing units available
+      - name: cmake
+        run: cmake -S ${{ github.workspace }} -B ${{ github.workspace }}/build -GNinja -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_RAPTOR=${{ matrix.raptor }}
+      - name: build
+        run: cd ${{ github.workspace }}/build && ninja -j`nproc`
+      - name: check for runtime linking errors in transmitter
+        run: ${{ github.workspace }}/build/examples/flute-transmitter --help
+      - name: check for runtime linking errors in receiver
+        run: ${{ github.workspace }}/build/examples/flute-receiver --help

.gitignore

@@ -33,4 +33,8 @@
 
 build/
 .idea
-cmake-build-debug/
+cmake-build-debug/
+html/
+.vscode/
+*.swp
+flute_download_*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "raptor"]
+	path = raptor
+	url = https://github.com/SteezyE/raptor.git

CMakeLists.txt

@@ -1,12 +1,16 @@
 cmake_minimum_required(VERSION 3.16)
 
-project (libflute VERSION 0.11.0)
+project (libflute VERSION 0.11.0 LANGUAGES C CXX)
 
 set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_C_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_C_STANDARD_REQUIRED True)
 
 set(CMAKE_CXX_FLAGS_DEBUG_INIT "-Wall -Wextra -Werror -g3")
+set(CMAKE_C_FLAGS_DEBUG_INIT "-Wall -Wextra -Werror -g3")
 set(CMAKE_CXX_FLAGS_RELEASE_INIT "-Wall -O3")
+set(CMAKE_C_FLAGS_RELEASE_INIT "-Wall -O3")
 
 find_package(Boost REQUIRED)
 find_package(spdlog REQUIRED)
@@ -15,11 +19,14 @@ find_package(OpenSSL REQUIRED)
 pkg_check_modules(TINYXML REQUIRED IMPORTED_TARGET tinyxml2)
 pkg_check_modules(NETLINK REQUIRED IMPORTED_TARGET libnl-3.0)
 
+option(ENABLE_RAPTOR "Enable support for Raptor FEC" ON)
+
 add_subdirectory(examples)
 
 include_directories(
     "${PROJECT_BINARY_DIR}"
     ${PROJECT_SOURCE_DIR}/include
+    ${PROJECT_SOURCE_DIR}/raptor
 
     SYSTEM
     ${PROJECT_SOURCE_DIR}/utils
@@ -29,25 +36,44 @@ include_directories(
 
 configure_file("include/Version.h.in" "Version.h")
 
-link_directories(
-    )
-
 set(CMAKE_CXX_CLANG_TIDY clang-tidy)
 
+if(ENABLE_RAPTOR)
+  message(STATUS "Compiling raptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF")
+  add_library(raptor STATIC)
+  target_sources(raptor
+    PRIVATE
+          raptor/bipartite.c raptor/decoder.c raptor/encoder.c raptor/galois.c raptor/gaussian.c raptor/pivoting.c raptor/random.c
+    PUBLIC
+          raptor/raptor.h
+  )
+  target_link_libraries(raptor LINK_PUBLIC m)
+else()
+  message(STATUS "Skipping raptor library for Raptor FEC support. To enable support build with -DENABLE_RAPTOR=ON")
+endif()
+
 add_library(flute "")
 target_sources(flute
   PRIVATE
-  src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp
-    utils/base64.cpp
+    src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp
+
   PUBLIC
     include/Receiver.h include/Transmitter.h include/File.h
+
   )
-target_include_directories(flute
-  PUBLIC
-    ${CMAKE_CURRENT_LIST_DIR}/include/
-  )
+target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/)
 
-#add_library(flute src/Receiver.cpp src/Receiver.h src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp)
+if(ENABLE_RAPTOR)
+  add_compile_definitions(RAPTOR_ENABLED)
+  target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/raptor/)
+  target_sources(flute
+    PRIVATE
+	  src/RaptorFEC.cpp
+	PUBLIC
+	  include/RaptorFEC.h
+  )
+  target_link_libraries(flute LINK_PUBLIC raptor)
+endif()
 
 target_link_libraries( flute
     LINK_PUBLIC
@@ -58,4 +84,3 @@ target_link_libraries( flute
     PkgConfig::TINYXML
     PkgConfig::NETLINK
 )
-

README.md

@@ -5,28 +5,37 @@
 
 ## Installation guide
 
-Installation of libflute consists of 3 simple steps:
+Installation of libflute consists of 2 simple steps:
 1. Getting the source code
-2. Build setup
-3. Building
+2. Building
 
 ### Step 1: Getting the source code
-````
-cd ~
-git clone https://github.com/5G-MAG/rt-libflute.git
-````
 
-### Step 2: Build setup
+This repository uses git submodules so it must be cloned with the following command
+
 ````
-cd libflute/
-mkdir build && cd build
-cmake -GNinja ..
+git clone --recurse-submodules https://github.com/5G-MAG/rt-libflute.git
 ````
 
-### Step 3: Building
+If you had checked it out before the submodules were added or ran git clone withou the `--recurse-submodules` argument then initialise and update the submodules by running
+
+```
+git submodule update --init
+```
+
+### Step 2: Building
 ````
-ninja
+cd rt-libflute/
+mkdir build
+cd build
+cmake ..
+cmake --build .
 ````
+(alternatively you can use the `build.sh` script to build in debug mode with raptor enabled)
+
+Build options:
+
+- "Enable Raptor": build with support for the raptor10-based forward error correction. On by default. Disable by passing the `-DENABLE_RAPTOR=OFF` option to cmake
 
 ## Usage
 
@@ -87,6 +96,19 @@ sudo setcap 'cap_net_admin=eip' ./flute-transmitter
 sudo setcap 'cap_net_admin=eip' ./flute-receiver
 ````
 
+### Optional: Forward Error Correction (FEC) for lossy environment
+
+To use forward error correction to overcome packet loss during transmission add the option `-f 1` to the transmitter. The receiver needs no such option, just make sure that both of them were properly built/rebuilt with raptor enabled in the build options (this is the default).
+
+To simulate packet loss over the loopback interface and test that FEC works, you can run the `setup_packet_loss_on_loopback` script, as root.
+
+Then start a transmission as you usually would. Make sure to use the default ip address, since the script will reroute this to go through the loopback interface.
+
+Finally revert the changes to your loopback interface and routing table with the `reset_loopback_settings` script.
+
 ## Documentation
 
 Documentation of the source code can be found at: https://5g-mag.github.io/rt-libflute/
+
+To generate it locally via doxygen run `doxygen` in the project root.
+Then to view it open the local file `html/index.html` in a browser

build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+rm -rf build
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=Debug -GNinja ..
+ninja -j`nproc`

examples/CMakeLists.txt

@@ -32,11 +32,13 @@ target_link_libraries( flute-transmitter
     config++
     flute
     pthread
+    m
 )
 target_link_libraries( flute-receiver
     LINK_PUBLIC
     spdlog::spdlog
     config++
     flute
     pthread
+    m
 )

examples/flute-receiver.cpp

@@ -32,6 +32,7 @@
 #include "Version.h"
 #include "Receiver.h"
 #include "File.h"
+#include "flute_types.h"
 
 
 using libconfig::Config;
@@ -52,6 +53,8 @@ static struct argp_option options[] = {  // NOLINT
      "Log verbosity: 0 = trace, 1 = debug, 2 = info, 3 = warn, 4 = error, 5 = "
      "critical, 6 = none. Default: 2.",
      0},
+    {"download-dir", 'd', "Download directory", 0 , "Directory in which to store downloaded files, defaults to the current directory otherwise", 0},
+    {"num-files", 'n', "Stop Receiving after n files", 0, "Stop the reception after n files have been received (default is to never stop)", 0},
     {nullptr, 0, nullptr, 0, nullptr, 0}};
 
 /**
@@ -64,6 +67,8 @@ struct ft_arguments {
   const char *aes_key = {};
   unsigned short mcast_port = 40085;
   unsigned log_level = 2;        /**< log level */
+  char *download_dir = nullptr;
+  unsigned nfiles = 0;        /**< log level */
   char **files;
 };
 
@@ -89,6 +94,12 @@ static auto parse_opt(int key, char *arg, struct argp_state *state) -> error_t {
     case 'l':
       arguments->log_level = static_cast<unsigned>(strtoul(arg, nullptr, 10));
       break;
+    case 'd':
+      arguments->download_dir = arg;
+      break;
+    case 'n':
+      arguments->nfiles = static_cast<unsigned>(strtoul(arg, nullptr, 10));
+      break;
     default:
       return ARGP_ERR_UNKNOWN;
   }
@@ -155,18 +166,39 @@ auto main(int argc, char **argv) -> int {
     }
 
     receiver.register_completion_callback(
-        [](std::shared_ptr<LibFlute::File> file) { //NOLINT
+        [&](std::shared_ptr<LibFlute::File> file) { //NOLINT
         spdlog::info("{} (TOI {}) has been received",
             file->meta().content_location, file->meta().toi);
-        FILE* fd = fopen(file->meta().content_location.c_str(), "wb");
-        fwrite(file->buffer(), 1, file->length(), fd);
-        fclose(fd);
+        char *buf = (char*) calloc(256,1);
+        char *fname = (char*) strrchr(file->meta().content_location.c_str(),'/');
+        if(!fname){
+          fname = (char*) file->meta().content_location.c_str();
+        } else {
+          fname++;
+        }
+        if (arguments.download_dir) {
+          snprintf(buf,256,"%s/%s",arguments.download_dir, fname);
+        } else {
+          snprintf(buf,256,"flute_download_%d-%s",file->meta().toi, fname);
+        }
+        FILE* fd = fopen(buf, "wb");
+        if (fd) {
+          fwrite(file->buffer(), 1, file->length(), fd);
+          fclose(fd);
+        } else {
+          spdlog::error("Error opening file {} to store received object",buf);
+        }
+        free(buf);
+        if (file->meta().toi == arguments.nfiles) {
+          spdlog::warn("{} file(s) received. Stopping reception",arguments.nfiles);
+          receiver.stop();
+        }
         });
 
     // Start the IO service
     io.run();
   } catch (std::exception ex ) {
-    spdlog::error("Exiting on unhandled exception: %s", ex.what());
+    spdlog::error("Exiting on unhandled exception: {}", ex.what());
   }
 
 exit: