From d8f64ef6bb813b4241b3ea45726fe121c75adbca Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 24 Nov 2022 11:14:19 +0100 Subject: [PATCH 01/61] reorder ipsec struct to fix compilation issues, change TSI in flute-transmitter to match receiver --- examples/flute-transmitter.cpp | 2 +- src/IpSec.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index 93538f3..4348009 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -173,7 +173,7 @@ auto main(int argc, char **argv) -> int { LibFlute::Transmitter transmitter( arguments.mcast_target, (short)arguments.mcast_port, - 0, + 16, arguments.mtu, arguments.rate_limit, io); diff --git a/src/IpSec.cpp b/src/IpSec.cpp index de500ab..167311e 100644 --- a/src/IpSec.cpp +++ b/src/IpSec.cpp @@ -95,8 +95,8 @@ namespace LibFlute::IpSec { xsinfo.mode = XFRM_MODE_TRANSPORT; struct { - struct xfrm_algo xa; char buf[512]; + struct xfrm_algo xa; } algo = {}; std::vector binary_key; From 05b04450dbd4c3cafab60c35cdaa2d05528e16fc Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 09:33:58 +0100 Subject: [PATCH 02/61] use switch cases for FEC type instead of if statements and added placeholders for future integration of Raptor FEC --- include/flute_types.h | 5 ++- src/AlcPacket.cpp | 35 ++++++++++------ src/EncodingSymbol.cpp | 91 +++++++++++++++++++++++++++++------------- src/File.cpp | 16 +++++--- 4 files changed, 100 insertions(+), 47 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index fe976e6..9046f85 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -39,10 +39,11 @@ namespace LibFlute { }; /** - * Error correction schemes + * Error correction schemes. From the registry for FEC schemes http://www.iana.org/assignments/rmt-fec-parameters (RFC 5052) */ enum class FecScheme { - CompactNoCode + CompactNoCode, + Raptor }; /** diff --git a/src/AlcPacket.cpp b/src/AlcPacket.cpp index d6e7fdc..7828fe3 100644 --- a/src/AlcPacket.cpp +++ b/src/AlcPacket.cpp @@ -110,19 +110,28 @@ LibFlute::AlcPacket::AlcPacket(char* data, size_t len) break; // ignored } case EXT_FTI: { - if (_fec_oti.encoding_id == FecScheme::CompactNoCode) { - if (hel != 4) { - throw "Invalid length for EXT_FTI header extension"; - } - _fec_oti.transfer_length = (uint64_t)(ntohs(*(uint16_t*)hdr_ptr)) << 32; - hdr_ptr += 2; - _fec_oti.transfer_length |= (uint64_t)(ntohl(*(uint32_t*)hdr_ptr)); - hdr_ptr += 4; - hdr_ptr += 2; // reserved - _fec_oti.encoding_symbol_length = ntohs(*(uint16_t*)hdr_ptr); - hdr_ptr += 2; - _fec_oti.max_source_block_length = ntohl(*(uint32_t*)hdr_ptr); - hdr_ptr += 4; + switch (_fec_oti.encoding_id) { + case FecScheme::CompactNoCode: + if (hel != 4) { + throw "Invalid length for EXT_FTI header extension for Compact No Code FEC scheme"; + } + _fec_oti.transfer_length = (uint64_t)(ntohs(*(uint16_t*)hdr_ptr)) << 32; + hdr_ptr += 2; + _fec_oti.transfer_length |= (uint64_t)(ntohl(*(uint32_t*)hdr_ptr)); + hdr_ptr += 4; + hdr_ptr += 2; // reserved + _fec_oti.encoding_symbol_length = ntohs(*(uint16_t*)hdr_ptr); + hdr_ptr += 2; + _fec_oti.max_source_block_length = ntohl(*(uint32_t*)hdr_ptr); + hdr_ptr += 4; + break; + case FecScheme::Raptor: + //TODO + throw "Raptor FEC support is still in progress"; + break; + default: + throw "Unsupported FEC scheme"; + break; } break; } diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index 5451534..73f4ac0 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and limitations // under the License. // +#include #include #include #include @@ -30,20 +31,34 @@ auto LibFlute::EncodingSymbol::from_payload(char* encoded_data, size_t data_len, throw "Only unencoded content is supported"; } - if (fec_oti.encoding_id == FecScheme::CompactNoCode) { - source_block_number = ntohs(*(uint16_t*)encoded_data); - encoded_data += 2; - encoding_symbol_id = ntohs(*(uint16_t*)encoded_data); - encoded_data += 2; - data_len -= 4; - } else { - throw "Only compact no-code FEC is supported"; + switch (fec_oti.encoding_id) { + case FecScheme::CompactNoCode: + source_block_number = ntohs(*(uint16_t*)encoded_data); + encoded_data += 2; + encoding_symbol_id = ntohs(*(uint16_t*)encoded_data); + encoded_data += 2; + data_len -= 4; + break; + case FecScheme::Raptor: + //TODO + spdlog::warn("EncodingSymbol::from_payload: Raptor FEC encoding implementation is still in progress"); + throw "EncodingSymbol::from_payload Raptor FEC is not done yet"; + break; + default: + throw "Invalid FEC encoding ID. Only 2 FEC types are currently supported: compact no-code or raptor"; + break; } int nof_symbols = std::ceil((float)data_len / (float)fec_oti.encoding_symbol_length); for (int i = 0; i < nof_symbols; i++) { - if (fec_oti.encoding_id == FecScheme::CompactNoCode) { - symbols.emplace_back(encoding_symbol_id, source_block_number, encoded_data, std::min(data_len, (size_t)fec_oti.encoding_symbol_length), fec_oti.encoding_id); + switch(fec_oti.encoding_id) { + case FecScheme::CompactNoCode: + symbols.emplace_back(encoding_symbol_id, source_block_number, encoded_data, std::min(data_len, (size_t)fec_oti.encoding_symbol_length), fec_oti.encoding_id); + break; + case FecScheme::Raptor: + //TODO + throw "Raptor support is still on its way"; + break; } encoded_data += fec_oti.encoding_symbol_length; encoding_symbol_id++; @@ -57,14 +72,22 @@ auto LibFlute::EncodingSymbol::to_payload(const std::vector& sym size_t len = 0; auto ptr = encoded_data; auto first_symbol = symbols.begin(); - if (fec_oti.encoding_id == FecScheme::CompactNoCode) { - *((uint16_t*)ptr) = htons(first_symbol->source_block_number()); - ptr += 2; - *((uint16_t*)ptr) = htons(first_symbol->id()); - ptr += 2; - len += 4; - } else { - throw "Only compact no-code FEC is supported"; + switch (fec_oti.encoding_id) { + case FecScheme::CompactNoCode: + *((uint16_t*)ptr) = htons(first_symbol->source_block_number()); + ptr += 2; + *((uint16_t*)ptr) = htons(first_symbol->id()); + ptr += 2; + len += 4; + break; + case FecScheme::Raptor: + //TODO + spdlog::warn("EncodingSymbol::to_payload: Raptor FEC encoding implementation is still in progress"); + throw "EncodingSymbol::to_payload Raptor FEC is not done yet"; + break; + default: + throw "Invalid FEC encoding ID. Only 2 FEC types are currently supported: compact no-code or raptor"; + break; } for (const auto& symbol : symbols) { @@ -79,19 +102,33 @@ auto LibFlute::EncodingSymbol::to_payload(const std::vector& sym } auto LibFlute::EncodingSymbol::decode_to(char* buffer, size_t max_length) const -> void { - if (_fec_scheme == FecScheme::CompactNoCode) { - if (_data_len <= max_length) { - memcpy(buffer, _encoded_data, _data_len); - } + switch (_fec_scheme) { + case FecScheme::CompactNoCode: + if (_data_len <= max_length) { + memcpy(buffer, _encoded_data, _data_len); + } + break; + case FecScheme::Raptor: + //TODO + spdlog::warn("EncodingSymbol::decode_to Raptor FEC encoding implementation is still in progress"); + throw "EncodingSymbol::decode_to Raptor FEC is not done yet"; + break; } } auto LibFlute::EncodingSymbol::encode_to(char* buffer, size_t max_length) const -> size_t { - if (_fec_scheme == FecScheme::CompactNoCode) { - if (_data_len <= max_length) { - memcpy(buffer, _encoded_data, _data_len); - return _data_len; - } + switch (_fec_scheme) { + case FecScheme::CompactNoCode: + if (_data_len <= max_length) { + memcpy(buffer, _encoded_data, _data_len); + return _data_len; + } + break; + case FecScheme::Raptor: + //TODO + spdlog::warn("EncodingSymbol::encode_to Raptor FEC encoding implementation is still in progress"); + throw "EncodingSymbol::encode_to Raptor FEC is not done yet"; + break; } return 0; } diff --git a/src/File.cpp b/src/File.cpp index 6f08020..ecee636 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -78,11 +78,17 @@ LibFlute::File::File(uint32_t toi, _meta.expires = expires; _meta.fec_oti = fec_oti; - // for no-code - if (_meta.fec_oti.encoding_id == FecScheme::CompactNoCode) { - _meta.fec_oti.transfer_length = length; - } else { - throw "Unsupported FEC scheme"; + switch (_meta.fec_oti.encoding_id) { + case FecScheme::CompactNoCode: + _meta.fec_oti.transfer_length = length; + break; + case FecScheme::Raptor: + //TODO + throw "Raptor FEC scheme is not done yet"; + break; + default: + throw "FEC scheme not supported or not yet implemented"; + break; } calculate_partitioning(); From 85d63deafbee4aac6ade0096dd07893829d3b4fc Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 10:39:42 +0100 Subject: [PATCH 03/61] added FEC scheme as cli argument for transmitter and receiver demos --- examples/flute-receiver.cpp | 12 ++++++++++++ examples/flute-transmitter.cpp | 11 +++++++++++ include/Receiver.h | 5 ++++- include/Transmitter.h | 3 +++ src/AlcPacket.cpp | 2 ++ src/EncodingSymbol.cpp | 2 +- src/File.cpp | 1 + src/Receiver.cpp | 3 ++- src/Transmitter.cpp | 10 ++++++---- 9 files changed, 42 insertions(+), 7 deletions(-) diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index 694d91a..23caf6e 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -32,6 +32,8 @@ #include "Version.h" #include "Receiver.h" #include "File.h" +#include "flute_types.h" + using libconfig::Config; @@ -52,6 +54,7 @@ static struct argp_option options[] = { // NOLINT "Log verbosity: 0 = trace, 1 = debug, 2 = info, 3 = warn, 4 = error, 5 = " "critical, 6 = none. Default: 2.", 0}, + {"fec", 'f', "FEC Scheme", 0, "Choose a scheme for Forward Error Correction. Compact No Code = 0, Raptor = 1 (default is 0)", 0}, {nullptr, 0, nullptr, 0, nullptr, 0}}; /** @@ -64,6 +67,7 @@ struct ft_arguments { const char *aes_key = {}; unsigned short mcast_port = 40085; unsigned log_level = 2; /**< log level */ + unsigned fec = 0; /**< log level */ char **files; }; @@ -89,6 +93,13 @@ static auto parse_opt(int key, char *arg, struct argp_state *state) -> error_t { case 'l': arguments->log_level = static_cast(strtoul(arg, nullptr, 10)); break; + case 'f': + arguments->fec = static_cast(strtoul(arg, nullptr, 10)); + if ( (arguments->fec | 1) != 1 ) { + spdlog::error("Invalid FEC scheme! Please pick either 0 (Compact No Code) or 1 (Raptor)"); + return ARGP_ERR_UNKNOWN; + } + break; default: return ARGP_ERR_UNKNOWN; } @@ -146,6 +157,7 @@ auto main(int argc, char **argv) -> int { arguments.mcast_target, (short)arguments.mcast_port, 16, + LibFlute::FecScheme(arguments.fec), io); // Configure IPSEC, if enabled diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index 4348009..5b382c4 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -31,6 +31,7 @@ #include "Version.h" #include "Transmitter.h" +#include "flute_types.h" using libconfig::Config; @@ -44,6 +45,7 @@ static char doc[] = "FLUTE/ALC transmitter demo"; // NOLINT static struct argp_option options[] = { // NOLINT {"target", 'm', "IP", 0, "Target multicast address (default: 238.1.1.95)", 0}, + {"fec", 'f', "FEC Scheme", 0, "Choose a scheme for Forward Error Correction. Compact No Code = 0, Raptor = 1 (default is 0)", 0}, {"port", 'p', "PORT", 0, "Target port (default: 40085)", 0}, {"mtu", 't', "BYTES", 0, "Path MTU to size ALC packets for (default: 1500)", 0}, {"rate-limit", 'r', "KBPS", 0, "Transmit rate limit (kbps), 0 = no limit, default: 1000 (1 Mbps)", 0}, @@ -65,6 +67,7 @@ struct ft_arguments { unsigned short mtu = 1500; uint32_t rate_limit = 1000; unsigned log_level = 2; /**< log level */ + unsigned fec = 0; /**< log level */ char **files; }; @@ -93,6 +96,13 @@ static auto parse_opt(int key, char *arg, struct argp_state *state) -> error_t { case 'l': arguments->log_level = static_cast(strtoul(arg, nullptr, 10)); break; + case 'f': + arguments->fec = static_cast(strtoul(arg, nullptr, 10)); + if ( (arguments->fec | 1) != 1 ) { + spdlog::error("Invalid FEC scheme ! Please pick either 0 (Compact No Code) or 1 (Raptor)"); + return ARGP_ERR_UNKNOWN; + } + break; case ARGP_KEY_NO_ARGS: argp_usage (state); case ARGP_KEY_ARG: @@ -176,6 +186,7 @@ auto main(int argc, char **argv) -> int { 16, arguments.mtu, arguments.rate_limit, + LibFlute::FecScheme(arguments.fec), io); // Configure IPSEC ESP, if enabled diff --git a/include/Receiver.h b/include/Receiver.h index 959c4c8..460f368 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -21,6 +21,7 @@ #include #include "File.h" #include "FileDeliveryTable.h" +#include "flute_types.h" namespace LibFlute { /** @@ -45,7 +46,7 @@ namespace LibFlute { * @param io_service Boost io_service to run the socket operations in (must be provided by the caller) */ Receiver( const std::string& iface, const std::string& address, - short port, uint64_t tsi, + short port, uint64_t tsi, FecScheme fec_scheme, boost::asio::io_service& io_service); /** @@ -104,5 +105,7 @@ namespace LibFlute { completion_callback_t _completion_cb = nullptr; bool _running = true; + + FecScheme _fec_scheme; }; }; diff --git a/include/Transmitter.h b/include/Transmitter.h index 1dd0565..e6d6635 100644 --- a/include/Transmitter.h +++ b/include/Transmitter.h @@ -23,6 +23,7 @@ #include "File.h" #include "AlcPacket.h" #include "FileDeliveryTable.h" +#include "flute_types.h" namespace LibFlute { /** @@ -51,6 +52,7 @@ namespace LibFlute { Transmitter( const std::string& address, short port, uint64_t tsi, unsigned short mtu, uint32_t rate_limit, + FecScheme _fec_scheme, boost::asio::io_service& io_service); /** @@ -124,6 +126,7 @@ namespace LibFlute { uint16_t _toi = 1; uint32_t _max_payload; + FecScheme _fec_scheme; FecOti _fec_oti; completion_callback_t _completion_cb = nullptr; diff --git a/src/AlcPacket.cpp b/src/AlcPacket.cpp index 7828fe3..baf8cca 100644 --- a/src/AlcPacket.cpp +++ b/src/AlcPacket.cpp @@ -17,6 +17,7 @@ #include #include #include "AlcPacket.h" +#include "spdlog/spdlog.h" LibFlute::AlcPacket::AlcPacket(char* data, size_t len) { @@ -127,6 +128,7 @@ LibFlute::AlcPacket::AlcPacket(char* data, size_t len) break; case FecScheme::Raptor: //TODO + spdlog::warn("Raptor FEC support is still in progress"); throw "Raptor FEC support is still in progress"; break; default: diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index 73f4ac0..37bb96f 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -13,12 +13,12 @@ // See the License for the specific language governing permissions and limitations // under the License. // -#include #include #include #include #include #include +#include "spdlog/spdlog.h" #include "EncodingSymbol.h" auto LibFlute::EncodingSymbol::from_payload(char* encoded_data, size_t data_len, const FecOti& fec_oti, ContentEncoding encoding) -> std::vector diff --git a/src/File.cpp b/src/File.cpp index ecee636..73958ae 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -84,6 +84,7 @@ LibFlute::File::File(uint32_t toi, break; case FecScheme::Raptor: //TODO + spdlog::warn("File.cpp - Raptor FEC scheme is not done yet"); throw "Raptor FEC scheme is not done yet"; break; default: diff --git a/src/Receiver.cpp b/src/Receiver.cpp index 6279325..e5ea008 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -22,10 +22,11 @@ LibFlute::Receiver::Receiver ( const std::string& iface, const std::string& address, - short port, uint64_t tsi, + short port, uint64_t tsi, FecScheme fec_scheme, boost::asio::io_service& io_service) : _socket(io_service) , _tsi(tsi) + , _fec_scheme(fec_scheme) , _mcast_address(address) { boost::asio::ip::udp::endpoint listen_endpoint( diff --git a/src/Transmitter.cpp b/src/Transmitter.cpp index 6d4d68d..5f2c204 100644 --- a/src/Transmitter.cpp +++ b/src/Transmitter.cpp @@ -21,9 +21,9 @@ #include "spdlog/spdlog.h" #include "Transmitter.h" #include "IpSec.h" -LibFlute::Transmitter::Transmitter ( const std::string& address, - short port, uint64_t tsi, unsigned short mtu, uint32_t rate_limit, - boost::asio::io_service& io_service) +LibFlute::Transmitter::Transmitter ( const std::string& address, short port, +uint64_t tsi, unsigned short mtu, uint32_t rate_limit, FecScheme fec_scheme, +boost::asio::io_service& io_service) : _endpoint(boost::asio::ip::address::from_string(address), port) , _socket(io_service, _endpoint.protocol()) , _fdt_timer(io_service) @@ -33,6 +33,7 @@ LibFlute::Transmitter::Transmitter ( const std::string& address, , _mtu(mtu) , _rate_limit(rate_limit) , _mcast_address(address) + ,_fec_scheme(fec_scheme) { _max_payload = mtu - 20 - // IPv4 header @@ -44,7 +45,8 @@ LibFlute::Transmitter::Transmitter ( const std::string& address, _socket.set_option(boost::asio::ip::multicast::enable_loopback(true)); _socket.set_option(boost::asio::ip::udp::socket::reuse_address(true)); - _fec_oti = FecOti{FecScheme::CompactNoCode, 0, _max_payload, max_source_block_length}; + //TODO: calculate max payload and max source block length properly based on FEC scheme + _fec_oti = FecOti{_fec_scheme, 0, _max_payload, max_source_block_length}; _fdt = std::make_unique(1, _fec_oti); _fdt_timer.expires_from_now(boost::posix_time::seconds(_fdt_repeat_interval)); From 79ab66cefa58ce63203c7533a283844e1b26bd66 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 1 Dec 2022 12:20:32 +0100 Subject: [PATCH 04/61] added build script --- build.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 build.sh diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..b738459 --- /dev/null +++ b/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +rm -rf build +mkdir build && cd build +cmake -GNinja .. +ninja From 38e931057ef37f50c50c128efc61b239f74faa41 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 1 Dec 2022 12:29:11 +0100 Subject: [PATCH 05/61] simplified installation guide --- README.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 562bcdb..2b3a43d 100644 --- a/README.md +++ b/README.md @@ -5,27 +5,19 @@ ## Installation guide -Installation of libflute consists of 3 simple steps: +Installation of libflute consists of 2 simple steps: 1. Getting the source code -2. Build setup -3. Building +2. Building ### Step 1: Getting the source code ```` -cd ~ git clone https://github.com/5G-MAG/rt-libflute.git ```` -### Step 2: Build setup +### Step 2: Building ```` -cd libflute/ -mkdir build && cd build -cmake -GNinja .. -```` - -### Step 3: Building -```` -ninja +cd rt-libflute/ +./build.sh ```` ## Usage From 818aec45498e4c6bbd3a35c93725c5fe4f6cc8bd Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 12:43:37 +0100 Subject: [PATCH 06/61] fixed md5 calculation to no longer emit deprecated openssl warning --- include/File.h | 2 ++ src/File.cpp | 29 +++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/include/File.h b/include/File.h index 081ef5f..c478c2a 100644 --- a/include/File.h +++ b/include/File.h @@ -163,4 +163,6 @@ namespace LibFlute { uint16_t _fdt_instance_id = 0; }; + + int calculate_md5(char *data, int length, unsigned char *return_sum); }; diff --git a/src/File.cpp b/src/File.cpp index 73958ae..7c52920 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include "base64.h" #include "spdlog/spdlog.h" @@ -67,8 +68,8 @@ LibFlute::File::File(uint32_t toi, _buffer = data; } - unsigned char md5[MD5_DIGEST_LENGTH]; - MD5((const unsigned char*)data, length, md5); + unsigned char md5[EVP_MAX_MD_SIZE]; + calculate_md5(data, length, &md5[0]); _meta.toi = toi; _meta.content_location = std::move(content_location); @@ -141,8 +142,8 @@ auto LibFlute::File::check_file_completion() -> void if (_complete && !_meta.content_md5.empty()) { //check MD5 sum - unsigned char md5[MD5_DIGEST_LENGTH]; - MD5((const unsigned char*)buffer(), length(), md5); + unsigned char md5[EVP_MAX_MD_SIZE]; + calculate_md5(buffer(),length(),&md5[0]); auto content_md5 = base64_decode(_meta.content_md5); if (memcmp(md5, content_md5.c_str(), MD5_DIGEST_LENGTH) != 0) { @@ -238,3 +239,23 @@ auto LibFlute::File::mark_completed(const std::vector& symbols, } } } + +int LibFlute::calculate_md5(char *data, int length, unsigned char *return_sum) +{ + EVP_MD_CTX* context = EVP_MD_CTX_new(); + const EVP_MD* md = EVP_md5(); + unsigned int md_len; + + EVP_DigestInit_ex2(context, md, NULL); + EVP_DigestUpdate(context, data, length); + EVP_DigestFinal_ex(context, return_sum, &md_len); + EVP_MD_CTX_free(context); + + char buf [EVP_MAX_MD_SIZE * 2] = {{0}}; + for (unsigned int i = 0 ; i < md_len ; ++i){ + sprintf(&buf[i*2], "%02x", return_sum[i]); + } + spdlog::debug("MD5 Digest is {}", buf); + + return md_len; +} \ No newline at end of file From dcb2085a8205c0a329ebdacf6845ba4eded91df8 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 13:10:31 +0100 Subject: [PATCH 07/61] document and cleanup calculate_md5() function --- include/File.h | 12 +++++++++++- src/File.cpp | 15 ++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/include/File.h b/include/File.h index c478c2a..d3d3451 100644 --- a/include/File.h +++ b/include/File.h @@ -164,5 +164,15 @@ namespace LibFlute { uint16_t _fdt_instance_id = 0; }; - int calculate_md5(char *data, int length, unsigned char *return_sum); + /** + * Calculate the md5 message digest + * + * @param input byte array whose md5 message digest shall be calculated + * @param length size of the input array + * @param result buffer to store the output of the md5 calculation. Make sure it is EVP_MAX_MD_SIZE bytes large + * + * @return length of the calculated md5 sum (should be 16 bytes for md5) + */ + int calculate_md5(char *input, int length, unsigned char *result); + }; diff --git a/src/File.cpp b/src/File.cpp index 7c52920..e45aa7e 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -69,7 +69,7 @@ LibFlute::File::File(uint32_t toi, } unsigned char md5[EVP_MAX_MD_SIZE]; - calculate_md5(data, length, &md5[0]); + calculate_md5(data, length, md5); _meta.toi = toi; _meta.content_location = std::move(content_location); @@ -143,7 +143,7 @@ auto LibFlute::File::check_file_completion() -> void if (_complete && !_meta.content_md5.empty()) { //check MD5 sum unsigned char md5[EVP_MAX_MD_SIZE]; - calculate_md5(buffer(),length(),&md5[0]); + calculate_md5(buffer(),length(),md5); auto content_md5 = base64_decode(_meta.content_md5); if (memcmp(md5, content_md5.c_str(), MD5_DIGEST_LENGTH) != 0) { @@ -240,22 +240,23 @@ auto LibFlute::File::mark_completed(const std::vector& symbols, } } -int LibFlute::calculate_md5(char *data, int length, unsigned char *return_sum) + int LibFlute::calculate_md5(char *input, int length, unsigned char *result) { + // simple implementation based on openssl docs (https://www.openssl.org/docs/man3.0/man3/EVP_DigestInit_ex.html) EVP_MD_CTX* context = EVP_MD_CTX_new(); const EVP_MD* md = EVP_md5(); unsigned int md_len; EVP_DigestInit_ex2(context, md, NULL); - EVP_DigestUpdate(context, data, length); - EVP_DigestFinal_ex(context, return_sum, &md_len); + EVP_DigestUpdate(context, input, length); + EVP_DigestFinal_ex(context, result, &md_len); EVP_MD_CTX_free(context); char buf [EVP_MAX_MD_SIZE * 2] = {{0}}; for (unsigned int i = 0 ; i < md_len ; ++i){ - sprintf(&buf[i*2], "%02x", return_sum[i]); + sprintf(&buf[i*2], "%02x", result[i]); } spdlog::debug("MD5 Digest is {}", buf); return md_len; -} \ No newline at end of file +} From 75a5175f3426053e9e4be787685ae552c7c4f3ad Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 1 Dec 2022 12:55:50 +0100 Subject: [PATCH 08/61] added freeRaptor submodule --- .gitmodules | 3 +++ freeRaptor | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 freeRaptor diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f18875c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "freeRaptor"] + path = freeRaptor + url = ./freeRaptor/ diff --git a/freeRaptor b/freeRaptor new file mode 160000 index 0000000..e21be84 --- /dev/null +++ b/freeRaptor @@ -0,0 +1 @@ +Subproject commit e21be8491dcfc9850129e7b456de5079f45cf65c From c96989b5bcabe7c4962236c48262cd314a9a9441 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 1 Dec 2022 13:10:02 +0100 Subject: [PATCH 09/61] modifying cmake and adding raptor fec files --- CMakeLists.txt | 11 ++++++++--- include/RaptorFEC.h | 12 ++++++++++++ src/RaptorFEC.cpp | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 include/RaptorFEC.h create mode 100644 src/RaptorFEC.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e94b64..1326784 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ add_subdirectory(examples) include_directories( "${PROJECT_BINARY_DIR}" ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/freeRaptor SYSTEM ${PROJECT_SOURCE_DIR}/utils @@ -37,14 +38,17 @@ set(CMAKE_CXX_CLANG_TIDY clang-tidy) add_library(flute "") target_sources(flute PRIVATE - src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp - utils/base64.cpp + src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp src/RaptorFEC.cpp + freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c + utils/base64.cpp PUBLIC - include/Receiver.h include/Transmitter.h include/File.h + include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h + freeRaptor/gf2matrix.h freeRaptor/raptor10.h freeRaptor/raptor_consts.h ) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/ + ${CMAKE_CURRENT_LIST_DIR}/freeRaptor ) #add_library(flute src/Receiver.cpp src/Receiver.h src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp) @@ -57,5 +61,6 @@ target_link_libraries( flute crypto PkgConfig::TINYXML PkgConfig::NETLINK + m ) diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h new file mode 100644 index 0000000..adb39b6 --- /dev/null +++ b/include/RaptorFEC.h @@ -0,0 +1,12 @@ +#ifndef __RAPTOR_FEC_H +#define __RAPTOR_FEC_H + +#include +#include + +void test_raptor(); + +// TODO: define encode function +// TODO: define decode function + +#endif diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp new file mode 100644 index 0000000..6c53d40 --- /dev/null +++ b/src/RaptorFEC.cpp @@ -0,0 +1,40 @@ +#include "RaptorFEC.h" + +void test_raptor() +{ + printf("Testing gf2matrix\n"); + + // Create a Raptor10 object and fill it w/ all known needed params + Raptor10 coder; + + coder.K = 22; + coder.Kmin = 1024; + coder.Kmax = 8192; + coder.Gmax = 10; + coder.Al = 4; + coder.N = 24; + coder.T = 4; + r10_compute_params(&coder); + printf("K=%u, S=%u, H=%u, L=%u\n", coder.K, coder.S, coder.H, coder.L); + + // Allocate and calculate the constraints matrix + gf2matrix A; + allocate_gf2matrix(&A, coder.L, coder.L); + r10_build_constraints_mat(&coder, &A); + + // LT encode + uint8_t enc_s[coder.L * coder.T]; + uint8_t src_s[coder.K * coder.T]; + r10_encode(src_s, enc_s, &coder, &A); + + // Now, enc_s should contain the encoded symbols + // Still, doesn't allow to decide the size of the symbols + + printf("Constraints matrix:\n"); + print_matrix(&A); + + return; +} + +// TODO: implement encode function +// TODO: implement decode function From 7fb40623b9f05f0b052ebe8633da309153fbfdfc Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 13:28:08 +0100 Subject: [PATCH 10/61] updated readme and .gitmodules for raptor submodule --- .gitmodules | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f18875c..41b8cb8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "freeRaptor"] path = freeRaptor - url = ./freeRaptor/ + url = https://github.com/Etua/freeRaptor.git diff --git a/README.md b/README.md index 2b3a43d..cb2e9fb 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Installation of libflute consists of 2 simple steps: ### Step 1: Getting the source code ```` -git clone https://github.com/5G-MAG/rt-libflute.git +git clone --recurse-submodules https://github.com/5G-MAG/rt-libflute.git ```` ### Step 2: Building From 9b7bb6a4823afe7c535db9611772ba5aafe59db8 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 13:36:49 +0100 Subject: [PATCH 11/61] point submodule towards free raptor dev branch --- .gitmodules | 1 + README.md | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.gitmodules b/.gitmodules index 41b8cb8..c1d3f9a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "freeRaptor"] path = freeRaptor url = https://github.com/Etua/freeRaptor.git + branch = dev diff --git a/README.md b/README.md index cb2e9fb..e6d4216 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,19 @@ Installation of libflute consists of 2 simple steps: 2. Building ### Step 1: Getting the source code + +This repository uses git submodules so it must be cloned with the following command + ```` git clone --recurse-submodules https://github.com/5G-MAG/rt-libflute.git ```` +If you had checked it out before the submodules were added or ran git clone withouth the argument then initialise and update the submodules by running + +``` +git submodule update --init +``` + ### Step 2: Building ```` cd rt-libflute/ From a0144804cd2c67fbc9cb6bef1b608c2df16ef983 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 1 Dec 2022 13:32:17 +0100 Subject: [PATCH 12/61] working on compiling/linking --- .gitignore | 2 +- examples/CMakeLists.txt | 3 +++ examples/flute-receiver.cpp | 3 ++- examples/flute-transmitter.cpp | 2 ++ include/RaptorFEC.h | 4 ++-- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c42b616..0db1ffe 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ build/ .idea -cmake-build-debug/ \ No newline at end of file +cmake-build-debug/ diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index bba302a..8151508 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -16,6 +16,7 @@ include_directories( "${PROJECT_BINARY_DIR}" ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/freeRaptor SYSTEM ${SPDLOG_INCLUDEDIR} @@ -32,6 +33,7 @@ target_link_libraries( flute-transmitter config++ flute pthread + m ) target_link_libraries( flute-receiver LINK_PUBLIC @@ -39,4 +41,5 @@ target_link_libraries( flute-receiver config++ flute pthread + m ) diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index 23caf6e..af09030 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -33,7 +33,7 @@ #include "Receiver.h" #include "File.h" #include "flute_types.h" - +#include "RaptorFEC.h" using libconfig::Config; @@ -136,6 +136,7 @@ auto main(int argc, char **argv) -> int { // Parse the arguments argp_parse(&argp, argc, argv, 0, nullptr, &arguments); + test_raptor(); // Set up logging std::string ident = "flute-receiver"; auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID | LOG_PERROR | LOG_CONS ); diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index 5b382c4..d863d02 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -32,6 +32,7 @@ #include "Version.h" #include "Transmitter.h" #include "flute_types.h" +#include "RaptorFEC.h" using libconfig::Config; @@ -142,6 +143,7 @@ auto main(int argc, char **argv) -> int { argp_parse(&argp, argc, argv, 0, nullptr, &arguments); + test_raptor(); // Set up logging std::string ident = "flute-transmitter"; auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID | LOG_PERROR | LOG_CONS ); diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h index adb39b6..3eebaf4 100644 --- a/include/RaptorFEC.h +++ b/include/RaptorFEC.h @@ -1,8 +1,8 @@ #ifndef __RAPTOR_FEC_H #define __RAPTOR_FEC_H -#include -#include +#include "gf2matrix.h" +#include "raptor10.h" void test_raptor(); From aa2d61c7ec89097a82595fda25ad061ede589bdf Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 1 Dec 2022 15:59:12 +0100 Subject: [PATCH 13/61] fixed linking/compiling using build script --- CMakeLists.txt | 10 +++------- build.sh | 7 +++++++ examples/CMakeLists.txt | 1 - freeRaptor | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1326784..d8bba3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,6 @@ add_subdirectory(examples) include_directories( "${PROJECT_BINARY_DIR}" ${PROJECT_SOURCE_DIR}/include - ${PROJECT_SOURCE_DIR}/freeRaptor SYSTEM ${PROJECT_SOURCE_DIR}/utils @@ -38,20 +37,17 @@ set(CMAKE_CXX_CLANG_TIDY clang-tidy) add_library(flute "") target_sources(flute PRIVATE - src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp src/RaptorFEC.cpp - freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c + src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp src/RaptorFEC.cpp src/gf2matrix.cpp src/raptor10.cpp src/raptor_consts.cpp utils/base64.cpp PUBLIC - include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h - freeRaptor/gf2matrix.h freeRaptor/raptor10.h freeRaptor/raptor_consts.h + include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h include/gf2matrix.h include/raptor10.h include/raptor_consts.h ) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/ - ${CMAKE_CURRENT_LIST_DIR}/freeRaptor ) -#add_library(flute src/Receiver.cpp src/Receiver.h src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp) +#add_library(flute src/Receiver.cpp src/Receiver.h src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c) target_link_libraries( flute LINK_PUBLIC diff --git a/build.sh b/build.sh index b738459..2bd3fe6 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,13 @@ #!/bin/bash +\cp freeRaptor/*.c src/ +\cp freeRaptor/*.h include/ +ls src/*.c | xargs -I{} mv {} {}pp rm -rf build mkdir build && cd build cmake -GNinja .. ninja +cd ../src +ls ../freeRaptor/*.c | awk -F "/" '{print $3"pp"}' | xargs rm -rf +cd ../include +ls ../freeRaptor/*.h | awk -F "/" '{print $3}' | xargs rm -rf diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 8151508..d8f3ded 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -16,7 +16,6 @@ include_directories( "${PROJECT_BINARY_DIR}" ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src - ${PROJECT_SOURCE_DIR}/freeRaptor SYSTEM ${SPDLOG_INCLUDEDIR} diff --git a/freeRaptor b/freeRaptor index e21be84..d3977ae 160000 --- a/freeRaptor +++ b/freeRaptor @@ -1 +1 @@ -Subproject commit e21be8491dcfc9850129e7b456de5079f45cf65c +Subproject commit d3977ae89e61a3782eecdff3827a48d563463bce From 09030427d0a728269452912651936ef30691fa3c Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 16:21:52 +0100 Subject: [PATCH 14/61] cmakelists integration of freeRaptor --- CMakeLists.txt | 32 +++++++++++++++++++++----------- build.sh | 7 ------- freeRaptor | 2 +- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d8bba3e..36abbde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,16 @@ cmake_minimum_required(VERSION 3.16) -project (libflute VERSION 0.11.0) +project (libflute VERSION 0.11.0 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) +set(CMAKE_C_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_C_STANDARD_REQUIRED True) set(CMAKE_CXX_FLAGS_DEBUG_INIT "-Wall -Wextra -Werror -g3") +set(CMAKE_C_FLAGS_DEBUG_INIT "-Wall -Wextra -Werror -g3") set(CMAKE_CXX_FLAGS_RELEASE_INIT "-Wall -O3") +set(CMAKE_C_FLAGS_RELEASE_INIT "-Wall -O3") find_package(Boost REQUIRED) find_package(spdlog REQUIRED) @@ -20,6 +24,7 @@ add_subdirectory(examples) include_directories( "${PROJECT_BINARY_DIR}" ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/freeRaptor SYSTEM ${PROJECT_SOURCE_DIR}/utils @@ -29,34 +34,39 @@ include_directories( configure_file("include/Version.h.in" "Version.h") -link_directories( - ) - set(CMAKE_CXX_CLANG_TIDY clang-tidy) +add_library(freeRaptor STATIC) +target_sources(freeRaptor + PRIVATE + freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c + PUBLIC + freeRaptor/gf2matrix.h freeRaptor/raptor10.h freeRaptor/raptor_consts.h +) +target_link_libraries(freeRaptor LINK_PUBLIC m) + add_library(flute "") target_sources(flute PRIVATE - src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp src/RaptorFEC.cpp src/gf2matrix.cpp src/raptor10.cpp src/raptor_consts.cpp - utils/base64.cpp + src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp src/RaptorFEC.cpp + PUBLIC - include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h include/gf2matrix.h include/raptor10.h include/raptor_consts.h + include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h + ) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/ + ${CMAKE_CURRENT_LIST_DIR}/freeRaptor/ ) - -#add_library(flute src/Receiver.cpp src/Receiver.h src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c) - target_link_libraries( flute LINK_PUBLIC + freeRaptor spdlog::spdlog z pthread crypto PkgConfig::TINYXML PkgConfig::NETLINK - m ) diff --git a/build.sh b/build.sh index 2bd3fe6..b738459 100755 --- a/build.sh +++ b/build.sh @@ -1,13 +1,6 @@ #!/bin/bash -\cp freeRaptor/*.c src/ -\cp freeRaptor/*.h include/ -ls src/*.c | xargs -I{} mv {} {}pp rm -rf build mkdir build && cd build cmake -GNinja .. ninja -cd ../src -ls ../freeRaptor/*.c | awk -F "/" '{print $3"pp"}' | xargs rm -rf -cd ../include -ls ../freeRaptor/*.h | awk -F "/" '{print $3}' | xargs rm -rf diff --git a/freeRaptor b/freeRaptor index d3977ae..b58805b 160000 --- a/freeRaptor +++ b/freeRaptor @@ -1 +1 @@ -Subproject commit d3977ae89e61a3782eecdff3827a48d563463bce +Subproject commit b58805bfae74e2f440e55113a25584c4da71d8db From df0cbb83b75b981d9f59998f668553305832a7cc Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 1 Dec 2022 16:54:25 +0100 Subject: [PATCH 15/61] make raptor compiliation optional and define a RAPTOR_ENABLED macro --- CMakeLists.txt | 22 ++++++++++++++++++++-- src/RaptorFEC.cpp | 3 +++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 36abbde..3cbfab9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,8 @@ find_package(OpenSSL REQUIRED) pkg_check_modules(TINYXML REQUIRED IMPORTED_TARGET tinyxml2) pkg_check_modules(NETLINK REQUIRED IMPORTED_TARGET libnl-3.0) +option(ENABLE_RAPTOR "Enable support for Raptor FEC" ON) + add_subdirectory(examples) include_directories( @@ -36,6 +38,12 @@ configure_file("include/Version.h.in" "Version.h") set(CMAKE_CXX_CLANG_TIDY clang-tidy) +if( ENABLE_RAPTOR) + +message(STATUS "Compiling freeRaptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF") + +add_compile_definitions(RAPTOR_ENABLED) + add_library(freeRaptor STATIC) target_sources(freeRaptor PRIVATE @@ -45,6 +53,12 @@ target_sources(freeRaptor ) target_link_libraries(freeRaptor LINK_PUBLIC m) +else() + +message(STATUS "Skipping freeRaptor library for Raptor FEC support. To enable support build with -DENABLE_RAPTOR=ON") + +endif() + add_library(flute "") target_sources(flute PRIVATE @@ -59,9 +73,13 @@ target_include_directories(flute ${CMAKE_CURRENT_LIST_DIR}/include/ ${CMAKE_CURRENT_LIST_DIR}/freeRaptor/ ) -target_link_libraries( flute + +if(ENABLE_RAPTOR) + target_link_libraries(flute LINK_PUBLIC freeRaptor) +endif() + + target_link_libraries( flute LINK_PUBLIC - freeRaptor spdlog::spdlog z pthread diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 6c53d40..0fa0569 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -1,7 +1,9 @@ + #include "RaptorFEC.h" void test_raptor() { +#ifdef RAPTOR_ENABLED printf("Testing gf2matrix\n"); // Create a Raptor10 object and fill it w/ all known needed params @@ -34,6 +36,7 @@ void test_raptor() print_matrix(&A); return; +#endif } // TODO: implement encode function From f17fb91cce0c14478e18907f781f26013882dc53 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Mon, 12 Dec 2022 17:07:58 +0100 Subject: [PATCH 16/61] add raptor submodule --- .gitmodules | 3 +++ raptor | 1 + 2 files changed, 4 insertions(+) create mode 160000 raptor diff --git a/.gitmodules b/.gitmodules index c1d3f9a..b9a661b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = freeRaptor url = https://github.com/Etua/freeRaptor.git branch = dev +[submodule "raptor"] + path = raptor + url = ./raptor/ diff --git a/raptor b/raptor new file mode 160000 index 0000000..d38cf5e --- /dev/null +++ b/raptor @@ -0,0 +1 @@ +Subproject commit d38cf5e3325ceb875edcd35d1ff4299dc7939431 From 77f472531a23e8677691cfcb7a19770903b7f57c Mon Sep 17 00:00:00 2001 From: SteezyE Date: Mon, 12 Dec 2022 17:11:32 +0100 Subject: [PATCH 17/61] add remote raptor repo to .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index b9a661b..a22dd3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,4 +4,4 @@ branch = dev [submodule "raptor"] path = raptor - url = ./raptor/ + url = https://github.com/yeliqseu/raptor.git From bdfc21eeae6aae53451fdec6a1c741ebdb31a7a1 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Mon, 12 Dec 2022 17:37:18 +0100 Subject: [PATCH 18/61] changed submodule to fork of raptor lib --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index a22dd3a..30a13f7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,4 +4,4 @@ branch = dev [submodule "raptor"] path = raptor - url = https://github.com/yeliqseu/raptor.git + url = https://github.com/SteezyE/raptor.git From c6a4eae3ca8a95aaf56cc5a91c64f1e6880f9bf2 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 15 Dec 2022 21:58:19 +0100 Subject: [PATCH 19/61] write donwloaded files to current directory, when building build in debug mode by default, small fixes in readme,cmakelists,gitignore --- .gitignore | 2 ++ CMakeLists.txt | 40 +++++++++++++--------------------- README.md | 5 ++++- build.sh | 4 ++-- examples/flute-receiver.cpp | 14 +++++++++++- examples/flute-transmitter.cpp | 9 ++++---- src/File.cpp | 2 +- src/Transmitter.cpp | 3 ++- 8 files changed, 43 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 0db1ffe..352082b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ build/ .idea cmake-build-debug/ +html/ +.vscode/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 3cbfab9..8d61b29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,24 +39,18 @@ configure_file("include/Version.h.in" "Version.h") set(CMAKE_CXX_CLANG_TIDY clang-tidy) if( ENABLE_RAPTOR) - -message(STATUS "Compiling freeRaptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF") - -add_compile_definitions(RAPTOR_ENABLED) - -add_library(freeRaptor STATIC) -target_sources(freeRaptor - PRIVATE - freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c - PUBLIC - freeRaptor/gf2matrix.h freeRaptor/raptor10.h freeRaptor/raptor_consts.h -) -target_link_libraries(freeRaptor LINK_PUBLIC m) - + message(STATUS "Compiling freeRaptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF") + add_compile_definitions(RAPTOR_ENABLED) + add_library(freeRaptor STATIC) + target_sources(freeRaptor + PRIVATE + freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c + PUBLIC + freeRaptor/gf2matrix.h freeRaptor/raptor10.h freeRaptor/raptor_consts.h + ) + target_link_libraries(freeRaptor LINK_PUBLIC m) else() - -message(STATUS "Skipping freeRaptor library for Raptor FEC support. To enable support build with -DENABLE_RAPTOR=ON") - + message(STATUS "Skipping freeRaptor library for Raptor FEC support. To enable support build with -DENABLE_RAPTOR=ON") endif() add_library(flute "") @@ -68,17 +62,14 @@ target_sources(flute include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h ) -target_include_directories(flute - PUBLIC - ${CMAKE_CURRENT_LIST_DIR}/include/ - ${CMAKE_CURRENT_LIST_DIR}/freeRaptor/ - ) +target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/) if(ENABLE_RAPTOR) + target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/freeRaptor/) target_link_libraries(flute LINK_PUBLIC freeRaptor) endif() - target_link_libraries( flute +target_link_libraries( flute LINK_PUBLIC spdlog::spdlog z @@ -86,5 +77,4 @@ endif() crypto PkgConfig::TINYXML PkgConfig::NETLINK -) - +) \ No newline at end of file diff --git a/README.md b/README.md index e6d4216..1e1cafc 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This repository uses git submodules so it must be cloned with the following comm git clone --recurse-submodules https://github.com/5G-MAG/rt-libflute.git ```` -If you had checked it out before the submodules were added or ran git clone withouth the argument then initialise and update the submodules by running +If you had checked it out before the submodules were added or ran git clone withou the `--recurse-submodules` argument then initialise and update the submodules by running ``` git submodule update --init @@ -91,3 +91,6 @@ sudo setcap 'cap_net_admin=eip' ./flute-receiver ## Documentation Documentation of the source code can be found at: https://5g-mag.github.io/rt-libflute/ + +To generate it locally via doxygen run `doxygen` in the project root. +Then to view it open the local file `html/index.html` in a browser diff --git a/build.sh b/build.sh index b738459..b3c4eab 100755 --- a/build.sh +++ b/build.sh @@ -2,5 +2,5 @@ rm -rf build mkdir build && cd build -cmake -GNinja .. -ninja +cmake -DCMAKE_BUILD_TYPE=Debug -GNinja .. +ninja -j`nproc` diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index af09030..2ea6a93 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -171,9 +171,21 @@ auto main(int argc, char **argv) -> int { [](std::shared_ptr file) { //NOLINT spdlog::info("{} (TOI {}) has been received", file->meta().content_location, file->meta().toi); - FILE* fd = fopen(file->meta().content_location.c_str(), "wb"); + char *buf = (char*) calloc(256,1); + char *fname = (char*) strrchr(file->meta().content_location.c_str(),'/'); + if(!fname){ + fname = (char*) file->meta().content_location.c_str(); + } else { + fname++; + } + snprintf(buf,256,"flute_download_%d-%s",file->meta().toi, fname); + FILE* fd = fopen(buf, "wb"); + if (!fd) { + spdlog::error("Error opening file {} to store received object",buf); + } fwrite(file->buffer(), 1, file->length(), fd); fclose(fd); + free(buf); }); // Start the IO service diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index d863d02..0af5674 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -201,11 +201,10 @@ auto main(int argc, char **argv) -> int { transmitter.register_completion_callback( [&files](uint32_t toi) { for (auto& file : files) { - if (file.toi == toi) { - spdlog::info("{} (TOI {}) has been transmitted", - file.location, file.toi); - // could free() the buffer here - } + if (file.toi == toi) { + spdlog::info("{} (TOI {}) has been transmitted", file.location,file.toi); + // could free() the buffer here + } } }); diff --git a/src/File.cpp b/src/File.cpp index e45aa7e..9520d1a 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -147,7 +147,7 @@ auto LibFlute::File::check_file_completion() -> void auto content_md5 = base64_decode(_meta.content_md5); if (memcmp(md5, content_md5.c_str(), MD5_DIGEST_LENGTH) != 0) { - spdlog::debug("MD5 mismatch for TOI {}, discarding", _meta.toi); + spdlog::warn("MD5 mismatch for TOI {}, discarding", _meta.toi); // MD5 mismatch, try again for (auto& block : _source_blocks) { diff --git a/src/Transmitter.cpp b/src/Transmitter.cpp index 5f2c204..4b793dc 100644 --- a/src/Transmitter.cpp +++ b/src/Transmitter.cpp @@ -153,6 +153,7 @@ auto LibFlute::Transmitter::send_next_packet() -> void } auto packet = std::make_shared(_tsi, file->meta().toi, file->meta().fec_oti, symbols, _max_payload, file->fdt_instance_id()); bytes_queued += packet->size(); + spdlog::debug("Queued ALC packet of {} bytes, containing {} symbols, for TOI {} , for transmission", packet->size(), symbols.size(), file->meta().toi ); _socket.async_send_to( boost::asio::buffer(packet->data(), packet->size()), _endpoint, @@ -161,7 +162,7 @@ auto LibFlute::Transmitter::send_next_packet() -> void std::size_t bytes_transferred) { if (error) { - spdlog::debug("sent_to error: {}", error.message()); + spdlog::debug("send_to error: {}", error.message()); } else { file->mark_completed(symbols, !error); if (file->complete()) { From 2428af11a3220d1c437ad1ce7bd27fad66b4f9c9 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 15 Dec 2022 22:17:00 +0100 Subject: [PATCH 20/61] flute-receiver: add cli argument for destination directory for downloaded files --- examples/flute-receiver.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index 2ea6a93..f46bb82 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -54,6 +54,7 @@ static struct argp_option options[] = { // NOLINT "Log verbosity: 0 = trace, 1 = debug, 2 = info, 3 = warn, 4 = error, 5 = " "critical, 6 = none. Default: 2.", 0}, + {"download-dir", 'd', "Download directory", 0 , "Directory in which to store downloaded files, defaults to the current directory otherwise", 0}, {"fec", 'f', "FEC Scheme", 0, "Choose a scheme for Forward Error Correction. Compact No Code = 0, Raptor = 1 (default is 0)", 0}, {nullptr, 0, nullptr, 0, nullptr, 0}}; @@ -67,6 +68,7 @@ struct ft_arguments { const char *aes_key = {}; unsigned short mcast_port = 40085; unsigned log_level = 2; /**< log level */ + char *download_dir = nullptr; unsigned fec = 0; /**< log level */ char **files; }; @@ -100,6 +102,9 @@ static auto parse_opt(int key, char *arg, struct argp_state *state) -> error_t { return ARGP_ERR_UNKNOWN; } break; + case 'd': + arguments->download_dir = arg; + break; default: return ARGP_ERR_UNKNOWN; } @@ -168,7 +173,7 @@ auto main(int argc, char **argv) -> int { } receiver.register_completion_callback( - [](std::shared_ptr file) { //NOLINT + [=](std::shared_ptr file) { //NOLINT spdlog::info("{} (TOI {}) has been received", file->meta().content_location, file->meta().toi); char *buf = (char*) calloc(256,1); @@ -178,13 +183,18 @@ auto main(int argc, char **argv) -> int { } else { fname++; } - snprintf(buf,256,"flute_download_%d-%s",file->meta().toi, fname); + if (arguments.download_dir) { + snprintf(buf,256,"%s/%s",arguments.download_dir, fname); + } else { + snprintf(buf,256,"flute_download_%d-%s",file->meta().toi, fname); + } FILE* fd = fopen(buf, "wb"); - if (!fd) { + if (fd) { + fwrite(file->buffer(), 1, file->length(), fd); + fclose(fd); + } else { spdlog::error("Error opening file {} to store received object",buf); } - fwrite(file->buffer(), 1, file->length(), fd); - fclose(fd); free(buf); }); From cf834d06845f77e8693a81942386dd4fb5a7b09d Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Wed, 21 Dec 2022 17:54:22 +0100 Subject: [PATCH 21/61] handle invalid file paths properly, increment the correct ptr in encoding symbols to_payload() functions --- examples/flute-transmitter.cpp | 13 ++++++++----- src/EncodingSymbol.cpp | 2 +- src/File.cpp | 15 ++++++++++++++- src/Transmitter.cpp | 24 +++++++++++++++--------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index 0af5674..fc75b4e 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -172,10 +172,11 @@ auto main(int argc, char **argv) -> int { std::ifstream file(arguments.files[j], std::ios::binary | std::ios::ate); std::streamsize size = file.tellg(); file.seekg(0, std::ios::beg); - - char* buffer = (char*)malloc(size); - file.read(buffer, size); - files.push_back(FsFile{ arguments.files[j], buffer, (size_t)size}); + if (size > 0) { + char* buffer = (char*)malloc(size); + file.read(buffer, size); + files.push_back(FsFile{ arguments.files[j], buffer, (size_t)size}); + } } // Create a Boost io_service @@ -216,8 +217,10 @@ auto main(int argc, char **argv) -> int { file.buffer, file.len ); - spdlog::info("Queued {} ({} bytes) for transmission, TOI is {}", + if (file.toi > 0) { + spdlog::info("Queued {} ({} bytes) for transmission, TOI is {}", file.location, file.len, file.toi); + } } // Start the io_service, and thus sending data diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index 37bb96f..9b5f76d 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -94,7 +94,7 @@ auto LibFlute::EncodingSymbol::to_payload(const std::vector& sym if (symbol.len() <= data_len) { auto symbol_len = symbol.encode_to(ptr, data_len); data_len -= symbol_len; - encoded_data += symbol_len; + ptr += symbol_len; len += symbol_len; } } diff --git a/src/File.cpp b/src/File.cpp index 9520d1a..8b9adc8 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -54,7 +54,13 @@ LibFlute::File::File(uint32_t toi, size_t length, bool copy_data) { + if (!data) { + spdlog::error("File pointer is null"); + throw "Invalid file"; + } + spdlog::debug("Creating File from data"); + if (copy_data) { spdlog::debug("Allocating buffer"); _buffer = (char*)malloc(length); @@ -69,7 +75,9 @@ LibFlute::File::File(uint32_t toi, } unsigned char md5[EVP_MAX_MD_SIZE]; - calculate_md5(data, length, md5); + if ( calculate_md5(data, length, md5) < 0 ){ + throw "Failed to calculate md5"; + } _meta.toi = toi; _meta.content_location = std::move(content_location); @@ -243,6 +251,11 @@ auto LibFlute::File::mark_completed(const std::vector& symbols, int LibFlute::calculate_md5(char *input, int length, unsigned char *result) { // simple implementation based on openssl docs (https://www.openssl.org/docs/man3.0/man3/EVP_DigestInit_ex.html) + if (!input || ! length) { + spdlog::debug("MD5 called for empty input (input {}, length {})", input, length); + return -1; + } + EVP_MD_CTX* context = EVP_MD_CTX_new(); const EVP_MD* md = EVP_md5(); unsigned int md_len; diff --git a/src/Transmitter.cpp b/src/Transmitter.cpp index 4b793dc..80c3b7a 100644 --- a/src/Transmitter.cpp +++ b/src/Transmitter.cpp @@ -98,18 +98,24 @@ auto LibFlute::Transmitter::send( size_t length) -> uint16_t { auto toi = _toi; + std::shared_ptr file; + try { + file = std::make_shared( + toi, + _fec_oti, + content_location, + content_type, + expires, + data, + length); + } catch (const char *e) { + spdlog::error("Failed to create File object for file {} : {}", content_location, e); + return -1; + } + _toi++; if (_toi == 0) _toi = 1; // clamp to >= 1 in case it wraps - auto file = std::make_shared( - toi, - _fec_oti, - content_location, - content_type, - expires, - data, - length); - _fdt->add(file->meta()); send_fdt(); _files.insert({toi, file}); From 940afb92071ee5ae0139ea6adaf88c1ad6c35a12 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 22 Dec 2022 10:05:30 +0100 Subject: [PATCH 22/61] abstract class definition for FEC object coding --- include/flute_types.h | 49 +++++++++++++++++++++++++++++++++++++++++-- src/RaptorFEC.cpp | 18 ++++++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 9046f85..2314794 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -47,12 +47,57 @@ namespace LibFlute { }; /** - * OTI values struct + * abstract class for FEC Object En/De-coding */ - struct FecOti { + class FecOti { + public: + FecScheme encoding_id; uint64_t transfer_length; uint32_t encoding_symbol_length; uint32_t max_source_block_length; + + /** + * @brief Attempt to decode a source block + * + * @param srcblk the source block that should be decoded + * @return whether or not the decoding was successful + */ + virtual bool check_source_block_completion(SourceBlock& srcblk); + + /** + * @brief Encode a source block into multiple symbols + * + * @param buffer a pointer to the buffer containing the data + * @param bytes_read a pointer to an integer to store the number of bytes read out of buffer + * @return a map of source blocks that the object has been encoded to + */ + virtual std::map create_blocks(char *buffer, int *bytes_read); + + + virtual void calculate_partioning(); + + private: + + }; + + class CompactNoCode : FecOti { + + + + } + + class RaptorFEC : FecOti { + + unsigned int F; // object size in bytes + unsigned int Al; // symbol alignment: 4 + unsigned int T; // symbol size in bytes + unsigned int Z; // number of source blocks + unsigned int N; // number of sub-blocks per source block + unsigned int K; // number of symbols in a source block + unsigned int P; // maximum payload size: 1420 for ipv4 over 802.3 + + } + }; diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 0fa0569..956fe6f 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -1,5 +1,7 @@ #include "RaptorFEC.h" +#include "flute_types.h" +#include "spdlog/spdlog.h" void test_raptor() { @@ -39,5 +41,17 @@ void test_raptor() #endif } -// TODO: implement encode function -// TODO: implement decode function + + void LibFlute::RaptorFEC::void calculate_partioning() { + // TODO + return; + } + + + bool LibFlute::RaptorFEC::check_source_block_completion(SourceBlock& srcblk) { + // TODO: try to decode srcblk using the symbols it contains... + } + + std::map LibFlute::RaptorFEC::create_blocks(char *buffer, int *bytes_read) { + // TODO: encode buffer into a number of symbols + } \ No newline at end of file From 1d327349a03b01ec69b121d7413e23d2fcb9c53f Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 22 Dec 2022 21:07:50 +0100 Subject: [PATCH 23/61] fixed abstract definition so it compiles, and reintroduced FecOti struct since its too difficult to directly replace it --- CMakeLists.txt | 4 +-- include/File.h | 14 ++-------- include/flute_types.h | 64 +++++++++++++++++++++++++++++++++---------- src/CompactNoCode.cpp | 29 ++++++++++++++++++++ src/File.cpp | 6 ++-- src/RaptorFEC.cpp | 25 ++++++++++++----- 6 files changed, 104 insertions(+), 38 deletions(-) create mode 100644 src/CompactNoCode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d61b29..493a5a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,7 @@ endif() add_library(flute "") target_sources(flute PRIVATE - src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp src/RaptorFEC.cpp + src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp src/RaptorFEC.cpp src/CompactNoCode.cpp PUBLIC include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h @@ -77,4 +77,4 @@ target_link_libraries( flute crypto PkgConfig::TINYXML PkgConfig::NETLINK -) \ No newline at end of file +) diff --git a/include/File.h b/include/File.h index d3d3451..be39635 100644 --- a/include/File.h +++ b/include/File.h @@ -21,6 +21,7 @@ #include "AlcPacket.h" #include "FileDeliveryTable.h" #include "EncodingSymbol.h" +#include "flute_types.h" namespace LibFlute { /** @@ -130,21 +131,10 @@ namespace LibFlute { void calculate_partitioning(); void create_blocks(); - struct SourceBlock { - bool complete = false; - struct Symbol { - char* data; - size_t length; - bool complete = false; - bool queued = false; - }; - std::map symbols; - }; - void check_source_block_completion(SourceBlock& block); void check_file_completion(); - std::map _source_blocks; + std::map _source_blocks; bool _complete = false;; diff --git a/include/flute_types.h b/include/flute_types.h index 2314794..80e30b2 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -13,8 +13,10 @@ // See the License for the specific language governing permissions and limitations // under the License. // +#include +#include +#include #pragma once - /** \mainpage LibFlute - ALC/FLUTE library * * The library contains two simple **example applications** as a starting point: @@ -46,16 +48,30 @@ namespace LibFlute { Raptor }; - /** - * abstract class for FEC Object En/De-coding - */ - class FecOti { - public: + struct Symbol { + char* data; + size_t length; + bool complete = false; + bool queued = false; + }; + struct SourceBlock { + bool complete = false; + std::map symbols; + }; + + struct FecOti { FecScheme encoding_id; uint64_t transfer_length; uint32_t encoding_symbol_length; uint32_t max_source_block_length; + }; + + /** + * abstract class for FEC Object En/De-coding + */ + class FecTransformer { + public: /** * @brief Attempt to decode a source block @@ -63,7 +79,7 @@ namespace LibFlute { * @param srcblk the source block that should be decoded * @return whether or not the decoding was successful */ - virtual bool check_source_block_completion(SourceBlock& srcblk); + virtual bool check_source_block_completion(SourceBlock& srcblk) = 0; /** * @brief Encode a source block into multiple symbols @@ -72,23 +88,43 @@ namespace LibFlute { * @param bytes_read a pointer to an integer to store the number of bytes read out of buffer * @return a map of source blocks that the object has been encoded to */ - virtual std::map create_blocks(char *buffer, int *bytes_read); + virtual std::map create_blocks(char *buffer, int *bytes_read) = 0; - virtual void calculate_partioning(); + virtual void calculate_partioning() = 0; + + }; + + class CompactNoCodeFEC : public FecTransformer { + + public: + + CompactNoCodeFEC(); - private: + virtual ~CompactNoCodeFEC(); + bool check_source_block_completion(SourceBlock& srcblk); + + std::map create_blocks(char *buffer, int *bytes_read); + + void calculate_partioning(); }; - class CompactNoCode : FecOti { + class RaptorFEC : public FecTransformer { + + public: + + RaptorFEC(); + virtual ~RaptorFEC(); + bool check_source_block_completion(SourceBlock& srcblk); - } + std::map create_blocks(char *buffer, int *bytes_read); + + void calculate_partioning(); - class RaptorFEC : FecOti { unsigned int F; // object size in bytes unsigned int Al; // symbol alignment: 4 @@ -98,6 +134,6 @@ namespace LibFlute { unsigned int K; // number of symbols in a source block unsigned int P; // maximum payload size: 1420 for ipv4 over 802.3 - } + }; }; diff --git a/src/CompactNoCode.cpp b/src/CompactNoCode.cpp new file mode 100644 index 0000000..d9765ef --- /dev/null +++ b/src/CompactNoCode.cpp @@ -0,0 +1,29 @@ +#include "flute_types.h" +#include "spdlog/spdlog.h" + +LibFlute::CompactNoCodeFEC::CompactNoCodeFEC() { +// this->encoding_id = LibFlute::FecScheme::CompactNoCode; +} + +LibFlute::CompactNoCodeFEC::CompactNoCodeFEC( uint64_t transfer_length, uint32_t encoding_symbol_length, uint32_t max_source_block_length) +{ +// this->encoding_symbol_length = encoding_symbol_length; +// this->transfer_length = transfer_length; +// this->max_source_block_length = max_source_block_length; +// this->encoding_id = LibFlute::FecScheme::CompactNoCode; +} + +LibFlute::CompactNoCodeFEC::~CompactNoCodeFEC() = default; + +bool LibFlute::CompactNoCodeFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk){ + return true; +} + +std::map LibFlute::CompactNoCodeFEC::create_blocks(char *buffer, int *bytes_read) { + std::map m; + return m; +} + +void LibFlute::CompactNoCodeFEC::calculate_partioning(){ + +} \ No newline at end of file diff --git a/src/File.cpp b/src/File.cpp index 8b9adc8..effbeb5 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -127,7 +127,7 @@ auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> voi throw "Encoding Symbol ID too high"; } - SourceBlock::Symbol& target_symbol = source_block.symbols[symbol.id()]; + LibFlute::Symbol& target_symbol = source_block.symbols[symbol.id()]; if (!target_symbol.complete) { symbol.decode_to(target_symbol.data, target_symbol.length); @@ -186,7 +186,7 @@ auto LibFlute::File::create_blocks() -> void size_t remaining_size = _meta.fec_oti.transfer_length; auto number = 0; while (remaining_size > 0) { - SourceBlock block; + LibFlute::SourceBlock block; auto symbol_id = 0; auto block_length = ( number < _nof_large_source_blocks ) ? _large_source_block_length : _small_source_block_length; @@ -194,7 +194,7 @@ auto LibFlute::File::create_blocks() -> void auto symbol_length = std::min(remaining_size, (size_t)_meta.fec_oti.encoding_symbol_length); assert(buffer_ptr + symbol_length <= _buffer + _meta.fec_oti.transfer_length); - SourceBlock::Symbol symbol{.data = buffer_ptr, .length = symbol_length, .complete = false}; + LibFlute::Symbol symbol{.data = buffer_ptr, .length = symbol_length, .complete = false}; block.symbols[ symbol_id++ ] = symbol; remaining_size -= symbol_length; diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 956fe6f..884e525 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -41,17 +41,28 @@ void test_raptor() #endif } +#ifdef RAPTOR_ENABLED - void LibFlute::RaptorFEC::void calculate_partioning() { +LibFlute::RaptorFEC::RaptorFEC() { + // this->encoding_id = LibFlute::FecScheme::Raptor; // TODO - return; - } +} +LibFlute::RaptorFEC::~RaptorFEC() = default; - bool LibFlute::RaptorFEC::check_source_block_completion(SourceBlock& srcblk) { +void LibFlute::RaptorFEC::calculate_partioning() { + // TODO +} + +bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { // TODO: try to decode srcblk using the symbols it contains... - } + return true; +} - std::map LibFlute::RaptorFEC::create_blocks(char *buffer, int *bytes_read) { +std::map LibFlute::RaptorFEC::create_blocks(char *buffer, int *bytes_read) { // TODO: encode buffer into a number of symbols - } \ No newline at end of file + std::map m; + return m; +} + +#endif \ No newline at end of file From 5e8941b3154626c27f63dc65910c4685b05261f8 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 29 Dec 2022 17:06:32 +0100 Subject: [PATCH 24/61] integreate FecTransformer structure into fdt and file classes --- examples/flute-receiver.cpp | 2 +- include/File.h | 4 +-- include/FileDeliveryTable.h | 6 ++-- include/Receiver.h | 5 ++- include/flute_types.h | 12 +++++++ src/CompactNoCode.cpp | 16 ++++++++-- src/File.cpp | 2 ++ src/FileDeliveryTable.cpp | 63 +++++++++++++++++++++++++++++++------ src/RaptorFEC.cpp | 11 +++++++ src/Receiver.cpp | 22 ++++++++++--- src/Transmitter.cpp | 4 ++- 11 files changed, 122 insertions(+), 25 deletions(-) diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index f46bb82..092a21e 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -163,7 +163,7 @@ auto main(int argc, char **argv) -> int { arguments.mcast_target, (short)arguments.mcast_port, 16, - LibFlute::FecScheme(arguments.fec), + // LibFlute::FecScheme(arguments.fec), io); // Configure IPSEC, if enabled diff --git a/include/File.h b/include/File.h index be39635..1378483 100644 --- a/include/File.h +++ b/include/File.h @@ -85,12 +85,12 @@ namespace LibFlute { /** * Get the FEC OTI values */ - const FecOti& fec_oti() const { return _meta.fec_oti; }; + FecOti& fec_oti() { return _meta.fec_oti; }; /** * Get the file metadata from its FDT entry */ - const LibFlute::FileDeliveryTable::FileEntry& meta() const { return _meta; }; + LibFlute::FileDeliveryTable::FileEntry& meta() { return _meta; }; /** * Timestamp of file reception diff --git a/include/FileDeliveryTable.h b/include/FileDeliveryTable.h index 57b98aa..8b585ad 100644 --- a/include/FileDeliveryTable.h +++ b/include/FileDeliveryTable.h @@ -46,7 +46,7 @@ namespace LibFlute { /** * Default destructor. */ - virtual ~FileDeliveryTable() {}; + virtual ~FileDeliveryTable(); /** * Get the FDT instance ID @@ -64,6 +64,7 @@ namespace LibFlute { std::string content_type; uint64_t expires; FecOti fec_oti; + FecTransformer *fec_transformer; }; /** @@ -74,7 +75,7 @@ namespace LibFlute { /** * Add a file entry */ - void add(const FileEntry& entry); + void add(FileEntry& entry); /** * Remove a file entry @@ -96,6 +97,7 @@ namespace LibFlute { std::vector _file_entries; FecOti _global_fec_oti; + FecTransformer *_fdt_fec_transformer = 0; uint64_t _expires; }; diff --git a/include/Receiver.h b/include/Receiver.h index 460f368..72f50f2 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -46,8 +46,7 @@ namespace LibFlute { * @param io_service Boost io_service to run the socket operations in (must be provided by the caller) */ Receiver( const std::string& iface, const std::string& address, - short port, uint64_t tsi, FecScheme fec_scheme, - boost::asio::io_service& io_service); + short port, uint64_t tsi, boost::asio::io_service& io_service); /** * Default destructor. @@ -106,6 +105,6 @@ namespace LibFlute { bool _running = true; - FecScheme _fec_scheme; + // FecScheme _fec_scheme; }; }; diff --git a/include/flute_types.h b/include/flute_types.h index 80e30b2..0a8527f 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -16,6 +16,7 @@ #include #include #include +#include "tinyxml2.h" #pragma once /** \mainpage LibFlute - ALC/FLUTE library * @@ -93,6 +94,10 @@ namespace LibFlute { virtual void calculate_partioning() = 0; + virtual bool parse_fdt_info(tinyxml2::XMLElement *file) = 0; + + virtual bool add_fdt_info(tinyxml2::XMLElement *file) = 0; + }; class CompactNoCodeFEC : public FecTransformer { @@ -109,6 +114,10 @@ namespace LibFlute { void calculate_partioning(); + bool parse_fdt_info(tinyxml2::XMLElement *file); + + bool add_fdt_info(tinyxml2::XMLElement *file); + }; class RaptorFEC : public FecTransformer { @@ -125,6 +134,9 @@ namespace LibFlute { void calculate_partioning(); + bool parse_fdt_info(tinyxml2::XMLElement *file); + + bool add_fdt_info(tinyxml2::XMLElement *file); unsigned int F; // object size in bytes unsigned int Al; // symbol alignment: 4 diff --git a/src/CompactNoCode.cpp b/src/CompactNoCode.cpp index d9765ef..c713a09 100644 --- a/src/CompactNoCode.cpp +++ b/src/CompactNoCode.cpp @@ -5,13 +5,13 @@ LibFlute::CompactNoCodeFEC::CompactNoCodeFEC() { // this->encoding_id = LibFlute::FecScheme::CompactNoCode; } -LibFlute::CompactNoCodeFEC::CompactNoCodeFEC( uint64_t transfer_length, uint32_t encoding_symbol_length, uint32_t max_source_block_length) -{ +// LibFlute::CompactNoCodeFEC::CompactNoCodeFEC( uint64_t transfer_length, uint32_t encoding_symbol_length, uint32_t max_source_block_length) +// { // this->encoding_symbol_length = encoding_symbol_length; // this->transfer_length = transfer_length; // this->max_source_block_length = max_source_block_length; // this->encoding_id = LibFlute::FecScheme::CompactNoCode; -} +// } LibFlute::CompactNoCodeFEC::~CompactNoCodeFEC() = default; @@ -26,4 +26,14 @@ std::map LibFlute::CompactNoCodeFEC::create_blo void LibFlute::CompactNoCodeFEC::calculate_partioning(){ +} + +bool LibFlute::CompactNoCodeFEC::parse_fdt_info(tinyxml2::XMLElement *file) { + // TODO + return true; +} + +bool LibFlute::CompactNoCodeFEC::add_fdt_info(tinyxml2::XMLElement *file) { + // TODO + return true; } \ No newline at end of file diff --git a/src/File.cpp b/src/File.cpp index effbeb5..7a84c55 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -90,8 +90,10 @@ LibFlute::File::File(uint32_t toi, switch (_meta.fec_oti.encoding_id) { case FecScheme::CompactNoCode: _meta.fec_oti.transfer_length = length; + _meta.fec_transformer = 0; break; case FecScheme::Raptor: + _meta.fec_transformer = new RaptorFEC(); // corresponding delete in FileDeliveryTable.cpp:remove() //TODO spdlog::warn("File.cpp - Raptor FEC scheme is not done yet"); throw "Raptor FEC scheme is not done yet"; diff --git a/src/FileDeliveryTable.cpp b/src/FileDeliveryTable.cpp index ec421d5..029da4e 100644 --- a/src/FileDeliveryTable.cpp +++ b/src/FileDeliveryTable.cpp @@ -14,7 +14,7 @@ // under the License. // #include "FileDeliveryTable.h" -#include "tinyxml2.h" +#include "tinyxml2.h" #include #include #include "spdlog/spdlog.h" @@ -24,8 +24,30 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, FecOti fec_ : _instance_id( instance_id ) , _global_fec_oti( fec_oti ) { + switch (fec_oti.encoding_id){ + case FecScheme::Raptor: + _fdt_fec_transformer = new RaptorFEC(); + break; + default: + _fdt_fec_transformer = 0; + break; + } +} + +LibFlute::FileDeliveryTable::~FileDeliveryTable() { + if (_fdt_fec_transformer) { + delete _fdt_fec_transformer; + _fdt_fec_transformer = 0; + } + for(auto &f : _file_entries) { + if (f.fec_transformer){ + delete f.fec_transformer; + f.fec_transformer = 0; + } + } } + LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffer, size_t len) : _instance_id( instance_id ) { @@ -99,6 +121,17 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffe encoding_id = strtoul(val, nullptr, 0); } + FecTransformer *fec_transformer = 0; + + switch (encoding_id){ + case (int) FecScheme::Raptor: + fec_transformer = new RaptorFEC(); // corresponding delete calls in Receiver.cpp and destuctor function + break; + default: + break; + } + + auto max_source_block_length = def_fec_max_source_block_length; val = file->Attribute("FEC-OTI-Maximum-Source-Block-Length"); if (val != nullptr) { @@ -110,6 +143,10 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffe if (val != nullptr) { encoding_symbol_length = strtoul(val, nullptr, 0); } + + if (fec_transformer && !fec_transformer->parse_fdt_info(file)) { + throw "Failed to parse fdt info for specific FEC data"; + } uint32_t expires = 0; auto cc = file->FirstChildElement("mbms2007:Cache-Control"); if (cc) { @@ -128,18 +165,19 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffe FileEntry fe{ toi, - std::string(content_location), - content_length, - std::string(content_md5), - std::string(content_type), - expires, - fec_oti + std::string(content_location), + content_length, + std::string(content_md5), + std::string(content_type), + expires, + fec_oti, + fec_transformer }; _file_entries.push_back(fe); } } -auto LibFlute::FileDeliveryTable::add(const FileEntry& fe) -> void +auto LibFlute::FileDeliveryTable::add(FileEntry& fe) -> void { _instance_id++; _file_entries.push_back(fe); @@ -147,8 +185,12 @@ auto LibFlute::FileDeliveryTable::add(const FileEntry& fe) -> void auto LibFlute::FileDeliveryTable::remove(uint32_t toi) -> void { - for (auto it = _file_entries.cbegin(); it != _file_entries.cend();) { + for (auto it = _file_entries.begin(); it != _file_entries.end();) { if (it->toi == toi) { + if (it->fec_transformer) { + delete it->fec_transformer; + it->fec_transformer = 0; + } it = _file_entries.erase(it); } else { ++it; @@ -176,6 +218,9 @@ auto LibFlute::FileDeliveryTable::to_string() const -> std::string { f->SetAttribute("Transfer-Length", (unsigned)file.fec_oti.transfer_length); f->SetAttribute("Content-MD5", file.content_md5.c_str()); f->SetAttribute("Content-Type", file.content_type.c_str()); + if(file.fec_transformer) { + file.fec_transformer->add_fdt_info(f); + } auto cc = doc.NewElement("mbms2007:Cache-Control"); auto exp = doc.NewElement("mbms2007:Expires"); exp->SetText(std::to_string(file.expires).c_str()); diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 884e525..5d43d63 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -65,4 +65,15 @@ std::map LibFlute::RaptorFEC::create_blocks(cha return m; } + +bool LibFlute::RaptorFEC::parse_fdt_info(tinyxml2::XMLElement *file) { + // TODO + return true; +} + +bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { + // TODO + return true; +} + #endif \ No newline at end of file diff --git a/src/Receiver.cpp b/src/Receiver.cpp index e5ea008..0e8ef7a 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -22,11 +22,9 @@ LibFlute::Receiver::Receiver ( const std::string& iface, const std::string& address, - short port, uint64_t tsi, FecScheme fec_scheme, - boost::asio::io_service& io_service) + short port, uint64_t tsi, boost::asio::io_service& io_service) : _socket(io_service) , _tsi(tsi) - , _fec_scheme(fec_scheme) , _mcast_address(address) { boost::asio::ip::udp::endpoint listen_endpoint( @@ -93,11 +91,15 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er auto file = _files[alc.toi()].get(); if (_files[alc.toi()]->complete()) { - for (auto it = _files.cbegin(); it != _files.cend();) + for (auto it = _files.begin(); it != _files.end();) { if (it->second.get() != file && it->second->meta().content_location == file->meta().content_location) { spdlog::debug("Replacing file with TOI {}", it->first); + if (it->second.get()->meta().fec_transformer){ + delete it->second.get()->meta().fec_transformer; + it->second.get()->meta().fec_transformer = 0; + } it = _files.erase(it); } else @@ -109,6 +111,10 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er spdlog::debug("File with TOI {} completed", alc.toi()); if (alc.toi() != 0 && _completion_cb) { _completion_cb(_files[alc.toi()]); + if (_files[alc.toi()]->meta().fec_transformer){ + delete _files[alc.toi()]->meta().fec_transformer; + _files[alc.toi()]->meta().fec_transformer = 0; + } _files.erase(alc.toi()); } @@ -162,6 +168,10 @@ auto LibFlute::Receiver::remove_expired_files(unsigned max_age) -> void { auto age = time(nullptr) - it->second->received_at(); if ( it->second->meta().content_location != "bootstrap.multipart" && age > max_age) { + if (it->second.get()->meta().fec_transformer){ + delete it->second.get()->meta().fec_transformer; + it->second.get()->meta().fec_transformer = 0; + } it = _files.erase(it); } else { ++it; @@ -175,6 +185,10 @@ auto LibFlute::Receiver::remove_file_with_content_location(const std::string& cl for (auto it = _files.cbegin(); it != _files.cend();) { if ( it->second->meta().content_location == cl) { + if (it->second.get()->meta().fec_transformer){ + delete it->second.get()->meta().fec_transformer; + it->second.get()->meta().fec_transformer = 0; + } it = _files.erase(it); } else { ++it; diff --git a/src/Transmitter.cpp b/src/Transmitter.cpp index 80c3b7a..9248ab8 100644 --- a/src/Transmitter.cpp +++ b/src/Transmitter.cpp @@ -77,9 +77,11 @@ auto LibFlute::Transmitter::seconds_since_epoch() -> uint64_t auto LibFlute::Transmitter::send_fdt() -> void { _fdt->set_expires(seconds_since_epoch() + _fdt_repeat_interval * 2); auto fdt = _fdt->to_string(); + auto fdt_fec_oti = _fec_oti; + fdt_fec_oti.encoding_id = FecScheme::CompactNoCode; // always send the FDT in "plaintext" auto file = std::make_shared( 0, - _fec_oti, + fdt_fec_oti, "", "", seconds_since_epoch() + _fdt_repeat_interval * 2, From f046ebe04dbdf2b643f57fa2fe3c78851ed6b113 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 29 Dec 2022 21:01:41 +0100 Subject: [PATCH 25/61] generation and parseing for raptor OTI, set codepoint in alc packets based on fec scheme in use --- include/flute_types.h | 33 +++++++++++++--- src/AlcPacket.cpp | 1 + src/CompactNoCode.cpp | 4 +- src/File.cpp | 18 +++++++-- src/RaptorFEC.cpp | 91 ++++++++++++++++++++++++++++++++++++++++--- src/Transmitter.cpp | 16 ++++++++ 6 files changed, 147 insertions(+), 16 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 0a8527f..a1fb45c 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -92,12 +92,19 @@ namespace LibFlute { virtual std::map create_blocks(char *buffer, int *bytes_read) = 0; - virtual void calculate_partioning() = 0; + virtual bool calculate_partitioning() = 0; virtual bool parse_fdt_info(tinyxml2::XMLElement *file) = 0; virtual bool add_fdt_info(tinyxml2::XMLElement *file) = 0; + uint32_t nof_source_symbols = 0; + uint32_t nof_source_blocks = 0; + uint32_t large_source_block_length = 0; + uint32_t small_source_block_length = 0; + uint32_t nof_large_source_blocks = 0; + + }; class CompactNoCodeFEC : public FecTransformer { @@ -112,18 +119,26 @@ namespace LibFlute { std::map create_blocks(char *buffer, int *bytes_read); - void calculate_partioning(); + bool calculate_partitioning(); bool parse_fdt_info(tinyxml2::XMLElement *file); bool add_fdt_info(tinyxml2::XMLElement *file); + uint32_t nof_source_symbols = 0; + uint32_t nof_source_blocks = 0; + uint32_t large_source_block_length = 0; + uint32_t small_source_block_length = 0; + uint32_t nof_large_source_blocks = 0; + }; class RaptorFEC : public FecTransformer { public: + RaptorFEC(unsigned int transfer_length, unsigned int max_payload, unsigned long target_sub_block_size); + RaptorFEC(); virtual ~RaptorFEC(); @@ -132,19 +147,27 @@ namespace LibFlute { std::map create_blocks(char *buffer, int *bytes_read); - void calculate_partioning(); + bool calculate_partitioning(); bool parse_fdt_info(tinyxml2::XMLElement *file); bool add_fdt_info(tinyxml2::XMLElement *file); + + uint32_t nof_source_symbols = 0; + uint32_t nof_source_blocks = 0; + uint32_t large_source_block_length = 0; + uint32_t small_source_block_length = 0; + uint32_t nof_large_source_blocks = 0; + unsigned int F; // object size in bytes - unsigned int Al; // symbol alignment: 4 + unsigned int Al = 4; // symbol alignment: 4 unsigned int T; // symbol size in bytes + unsigned long W = 1024*1024; // target on sub block size (arbitrarily set default to 1 MB) unsigned int Z; // number of source blocks unsigned int N; // number of sub-blocks per source block unsigned int K; // number of symbols in a source block - unsigned int P; // maximum payload size: 1420 for ipv4 over 802.3 + unsigned int P; // maximum payload size: e.g. 1436 for ipv4 over 802.3 }; diff --git a/src/AlcPacket.cpp b/src/AlcPacket.cpp index baf8cca..4741141 100644 --- a/src/AlcPacket.cpp +++ b/src/AlcPacket.cpp @@ -185,6 +185,7 @@ LibFlute::AlcPacket::AlcPacket(uint16_t tsi, uint16_t toi, LibFlute::FecOti fec_ lct_header->version = 1; lct_header->half_word_flag = 1; lct_header->lct_header_len = lct_header_len; + lct_header->codepoint = (uint8_t) fec_oti.encoding_id; auto hdr_ptr = _buffer + 4; auto payload_ptr = _buffer + 4 * lct_header_len; diff --git a/src/CompactNoCode.cpp b/src/CompactNoCode.cpp index c713a09..afb584c 100644 --- a/src/CompactNoCode.cpp +++ b/src/CompactNoCode.cpp @@ -24,8 +24,8 @@ std::map LibFlute::CompactNoCodeFEC::create_blo return m; } -void LibFlute::CompactNoCodeFEC::calculate_partioning(){ - +bool LibFlute::CompactNoCodeFEC::calculate_partitioning(){ + return false; } bool LibFlute::CompactNoCodeFEC::parse_fdt_info(tinyxml2::XMLElement *file) { diff --git a/src/File.cpp b/src/File.cpp index 7a84c55..6bf2efa 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -93,10 +93,14 @@ LibFlute::File::File(uint32_t toi, _meta.fec_transformer = 0; break; case FecScheme::Raptor: - _meta.fec_transformer = new RaptorFEC(); // corresponding delete in FileDeliveryTable.cpp:remove() - //TODO + _meta.fec_oti.transfer_length = length; + _meta.fec_transformer = new RaptorFEC(length, fec_oti.encoding_symbol_length, 1024*1024); // corresponding delete in FileDeliveryTable.cpp:remove() + // set W (target on the sub block size, to be fairly large - 1 MB for now) + spdlog::warn("File.cpp - Raptor FEC scheme is not done yet"); - throw "Raptor FEC scheme is not done yet"; + + // TODO... what else is missing here? + break; default: throw "FEC scheme not supported or not yet implemented"; @@ -173,6 +177,14 @@ auto LibFlute::File::check_file_completion() -> void auto LibFlute::File::calculate_partitioning() -> void { + if (0 && _meta.fec_transformer && _meta.fec_transformer->calculate_partitioning()){ + _nof_source_symbols = _meta.fec_transformer->nof_source_symbols; + _nof_source_blocks = _meta.fec_transformer->nof_source_blocks; + _large_source_block_length = _meta.fec_transformer->large_source_block_length; + _small_source_block_length = _meta.fec_transformer->small_source_block_length; + _nof_large_source_blocks = _meta.fec_transformer->nof_large_source_blocks; + return; + } // Calculate source block partitioning (RFC5052 9.1) _nof_source_symbols = ceil((double)_meta.fec_oti.transfer_length / (double)_meta.fec_oti.encoding_symbol_length); _nof_source_blocks = ceil((double)_nof_source_symbols / (double)_meta.fec_oti.max_source_block_length); diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 5d43d63..e3c4bee 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -43,15 +43,48 @@ void test_raptor() #ifdef RAPTOR_ENABLED -LibFlute::RaptorFEC::RaptorFEC() { - // this->encoding_id = LibFlute::FecScheme::Raptor; - // TODO +LibFlute::RaptorFEC::RaptorFEC(){} + + +LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_payload, unsigned long target_sub_block_size) + : F(transfer_length) + , P(max_payload) + , W(target_sub_block_size) +{ + Al = 4; + calculate_partitioning(); } LibFlute::RaptorFEC::~RaptorFEC() = default; -void LibFlute::RaptorFEC::calculate_partioning() { - // TODO +bool LibFlute::RaptorFEC::calculate_partitioning() { + // TODO: print debug statements and test + double G = fmin( fmin(ceil((double)P*1024/(double)F), (double)P/(double)Al), 10.0f); + spdlog::debug("double G = fmin( fmin(ceil((double)P*1024/F), (double)P/(double)Al), 10.0f"); + spdlog::debug("G = {} = min( ceil({}*1024/{}), {}/{}, 10.0f)",G,P,F,P,Al); + + T = (unsigned int) floor((double)P/(double)(Al*G)) * Al; + spdlog::debug("T = (unsigned int) floor((double)P/(double)(Al*G)) * Al"); + spdlog::debug("T = {} = floor({}/({}*{})) * {}",T,P,Al,G,Al); + + assert(T % Al == 0); // Symbol size T should be a multiple of symbol alignment parameter Al + + double Kt = ceil((double)F/(double)T); + spdlog::debug("double Kt = ceil((double)F/(double)T)"); + spdlog::debug("Kt = {} = ceil({}/{})",Kt,F,T); + + Z = (unsigned int) ceil(Kt/8192); + spdlog::debug("Z = (unsigned int) ceil(Kt/8192)"); + spdlog::debug("Z = {} = ceil({}/8192)",Z,Kt); + + + N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al ); + spdlog::warn("N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al )"); + spdlog::warn("N = {} = min( ceil( ceil({}/{}) * {}/{} ) , {}/{} )",N,Kt,Z,T,W,T,Al); + + //TODO set the values that the File class needs... + + return true; } bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { @@ -68,11 +101,57 @@ std::map LibFlute::RaptorFEC::create_blocks(cha bool LibFlute::RaptorFEC::parse_fdt_info(tinyxml2::XMLElement *file) { // TODO + + const char* val = 0; + val = file->Attribute("Transfer-Length"); + if (val != nullptr) { + F = strtoul(val, nullptr, 0); + } else { + throw "Required field \"Transfer-Length\" is missing for an object in the FDT"; + } + + val = file->Attribute("FEC-OTI-Number-Of-Source-Blocks"); + if (val != nullptr) { + Z = strtoul(val, nullptr, 0); + } else { + throw "Required field \"FEC-OTI-Number-Of-Source-Blocks\" is missing for an object in the FDT"; + } + + val = file->Attribute("FEC-OTI-Number-Of-Sub-Blocks"); + if (val != nullptr) { + N = strtoul(val, nullptr, 0); + } else { + throw "Required field \"FEC-OTI-Number-Of-Sub-Blocks\" is missing for an object in the FDT"; + } + + val = file->Attribute("FEC-OTI-Encoding-Symbol-Length"); + if (val != nullptr) { + T = strtoul(val, nullptr, 0); + } else { + throw "Required field \"FEC-OTI-Encoding-Symbol-Length\" is missing for an object in the FDT"; + } + + val = file->Attribute("FEC-OTI-Symbol-Alignment-Parameter"); + if (val != nullptr) { + Al = strtoul(val, nullptr, 0); + } else { + throw "Required field \"FEC-OTI-Symbol-Alignment-Parameter\" is missing for an object in the FDT"; + } + + //TODO: calculate other relevant values from these + return true; } bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { - // TODO + //TODO: do we need transfer length too? Does it change based on FecScheme? + file->SetAttribute("FEC-OTI-FEC-Encoding-ID", (unsigned) FecScheme::Raptor); + file->SetAttribute("FEC-OTI-Encoding-Symbol-Length", T); + file->SetAttribute("FEC-OTI-Symbol-Alignment-Parameter", Al); + file->SetAttribute("FEC-OTI-Number-Of-Source-Blocks", Z); + file->SetAttribute("FEC-OTI-Number-Of-Sub-Blocks", N); + file->SetAttribute("FEC-OTI-Symbol-Alignment-Parameter", Al); + return true; } diff --git a/src/Transmitter.cpp b/src/Transmitter.cpp index 9248ab8..d676133 100644 --- a/src/Transmitter.cpp +++ b/src/Transmitter.cpp @@ -42,6 +42,16 @@ boost::asio::io_service& io_service) 4; // SBN and ESI for compact no-code FEC uint32_t max_source_block_length = 64; + switch(_fec_scheme) { + case FecScheme::Raptor: + if (_max_payload % 4) { + _max_payload -= (_max_payload % 4); // must be divisible by Al = 4 + } + break; + default: + break; + } + _socket.set_option(boost::asio::ip::multicast::enable_loopback(true)); _socket.set_option(boost::asio::ip::udp::socket::reuse_address(true)); @@ -79,6 +89,12 @@ auto LibFlute::Transmitter::send_fdt() -> void { auto fdt = _fdt->to_string(); auto fdt_fec_oti = _fec_oti; fdt_fec_oti.encoding_id = FecScheme::CompactNoCode; // always send the FDT in "plaintext" + fdt_fec_oti.encoding_symbol_length = _mtu - + 20 - // IPv4 header + 8 - // UDP header + 32 - // ALC Header with EXT_FDT and EXT_FTI + 4; // SBN and ESI for compact no-code FEC + fdt_fec_oti.max_source_block_length = 64; auto file = std::make_shared( 0, fdt_fec_oti, From a2d0b63bc5a93e204c59c9ec761e6149197fb118 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Fri, 30 Dec 2022 14:42:01 +0100 Subject: [PATCH 26/61] Switch freeRaptor to raptor --- .gitmodules | 4 ---- CMakeLists.txt | 30 ++++++++++++------------- examples/flute-receiver.cpp | 2 -- examples/flute-transmitter.cpp | 4 +--- freeRaptor | 1 - include/RaptorFEC.h | 12 ---------- src/RaptorFEC.cpp | 40 ---------------------------------- 7 files changed, 16 insertions(+), 77 deletions(-) delete mode 160000 freeRaptor delete mode 100644 include/RaptorFEC.h diff --git a/.gitmodules b/.gitmodules index 30a13f7..8e8412b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "freeRaptor"] - path = freeRaptor - url = https://github.com/Etua/freeRaptor.git - branch = dev [submodule "raptor"] path = raptor url = https://github.com/SteezyE/raptor.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 493a5a2..94495b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ add_subdirectory(examples) include_directories( "${PROJECT_BINARY_DIR}" ${PROJECT_SOURCE_DIR}/include - ${PROJECT_SOURCE_DIR}/freeRaptor + ${PROJECT_SOURCE_DIR}/raptor SYSTEM ${PROJECT_SOURCE_DIR}/utils @@ -38,19 +38,19 @@ configure_file("include/Version.h.in" "Version.h") set(CMAKE_CXX_CLANG_TIDY clang-tidy) -if( ENABLE_RAPTOR) - message(STATUS "Compiling freeRaptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF") +if(ENABLE_RAPTOR) + message(STATUS "Compiling raptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF") add_compile_definitions(RAPTOR_ENABLED) - add_library(freeRaptor STATIC) - target_sources(freeRaptor + add_library(raptor STATIC) + target_sources(raptor PRIVATE - freeRaptor/gf2matrix.c freeRaptor/raptor10.c freeRaptor/raptor_consts.c - PUBLIC - freeRaptor/gf2matrix.h freeRaptor/raptor10.h freeRaptor/raptor_consts.h + raptor/bipartite.c raptor/decoder.c raptor/encoder.c raptor/galois.c raptor/gaussian.c raptor/pivoting.c + PUBLIC + raptor/raptor.h ) - target_link_libraries(freeRaptor LINK_PUBLIC m) + target_link_libraries(raptor LINK_PUBLIC m) else() - message(STATUS "Skipping freeRaptor library for Raptor FEC support. To enable support build with -DENABLE_RAPTOR=ON") + message(STATUS "Skipping raptor library for Raptor FEC support. To enable support build with -DENABLE_RAPTOR=ON") endif() add_library(flute "") @@ -59,15 +59,15 @@ target_sources(flute src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp src/RaptorFEC.cpp src/CompactNoCode.cpp PUBLIC - include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h + include/Receiver.h include/Transmitter.h include/File.h ) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/) -if(ENABLE_RAPTOR) - target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/freeRaptor/) - target_link_libraries(flute LINK_PUBLIC freeRaptor) -endif() +#if(ENABLE_RAPTOR) +# target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/raptor/) +# target_link_libraries(flute LINK_PUBLIC raptor) +#endif() target_link_libraries( flute LINK_PUBLIC diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index 092a21e..f63fa96 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -33,7 +33,6 @@ #include "Receiver.h" #include "File.h" #include "flute_types.h" -#include "RaptorFEC.h" using libconfig::Config; @@ -141,7 +140,6 @@ auto main(int argc, char **argv) -> int { // Parse the arguments argp_parse(&argp, argc, argv, 0, nullptr, &arguments); - test_raptor(); // Set up logging std::string ident = "flute-receiver"; auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID | LOG_PERROR | LOG_CONS ); diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index fc75b4e..d544802 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -32,7 +32,6 @@ #include "Version.h" #include "Transmitter.h" #include "flute_types.h" -#include "RaptorFEC.h" using libconfig::Config; @@ -142,8 +141,7 @@ auto main(int argc, char **argv) -> int { arguments.mcast_target = "238.1.1.95"; argp_parse(&argp, argc, argv, 0, nullptr, &arguments); - - test_raptor(); + // Set up logging std::string ident = "flute-transmitter"; auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID | LOG_PERROR | LOG_CONS ); diff --git a/freeRaptor b/freeRaptor deleted file mode 160000 index b58805b..0000000 --- a/freeRaptor +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b58805bfae74e2f440e55113a25584c4da71d8db diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h deleted file mode 100644 index 3eebaf4..0000000 --- a/include/RaptorFEC.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __RAPTOR_FEC_H -#define __RAPTOR_FEC_H - -#include "gf2matrix.h" -#include "raptor10.h" - -void test_raptor(); - -// TODO: define encode function -// TODO: define decode function - -#endif diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index e3c4bee..9bf8691 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -1,46 +1,6 @@ - -#include "RaptorFEC.h" #include "flute_types.h" #include "spdlog/spdlog.h" -void test_raptor() -{ -#ifdef RAPTOR_ENABLED - printf("Testing gf2matrix\n"); - - // Create a Raptor10 object and fill it w/ all known needed params - Raptor10 coder; - - coder.K = 22; - coder.Kmin = 1024; - coder.Kmax = 8192; - coder.Gmax = 10; - coder.Al = 4; - coder.N = 24; - coder.T = 4; - r10_compute_params(&coder); - printf("K=%u, S=%u, H=%u, L=%u\n", coder.K, coder.S, coder.H, coder.L); - - // Allocate and calculate the constraints matrix - gf2matrix A; - allocate_gf2matrix(&A, coder.L, coder.L); - r10_build_constraints_mat(&coder, &A); - - // LT encode - uint8_t enc_s[coder.L * coder.T]; - uint8_t src_s[coder.K * coder.T]; - r10_encode(src_s, enc_s, &coder, &A); - - // Now, enc_s should contain the encoded symbols - // Still, doesn't allow to decide the size of the symbols - - printf("Constraints matrix:\n"); - print_matrix(&A); - - return; -#endif -} - #ifdef RAPTOR_ENABLED LibFlute::RaptorFEC::RaptorFEC(){} From 909cc3ccbee25f19d50c1e7adb71f99843e13760 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Fri, 30 Dec 2022 20:56:42 +0100 Subject: [PATCH 27/61] use 256kB for target sub block size instead of 1MB --- include/flute_types.h | 2 +- src/EncodingSymbol.cpp | 6 +----- src/File.cpp | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index a1fb45c..ec350eb 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -163,7 +163,7 @@ namespace LibFlute { unsigned int F; // object size in bytes unsigned int Al = 4; // symbol alignment: 4 unsigned int T; // symbol size in bytes - unsigned long W = 1024*1024; // target on sub block size (arbitrarily set default to 1 MB) + unsigned long W = 256*1024; // target on sub block size (arbitrarily set to maximum recommended value of 256 kB) unsigned int Z; // number of source blocks unsigned int N; // number of sub-blocks per source block unsigned int K; // number of symbols in a source block diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index 9b5f76d..6175dd3 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -33,17 +33,13 @@ auto LibFlute::EncodingSymbol::from_payload(char* encoded_data, size_t data_len, switch (fec_oti.encoding_id) { case FecScheme::CompactNoCode: + case FecScheme::Raptor: source_block_number = ntohs(*(uint16_t*)encoded_data); encoded_data += 2; encoding_symbol_id = ntohs(*(uint16_t*)encoded_data); encoded_data += 2; data_len -= 4; break; - case FecScheme::Raptor: - //TODO - spdlog::warn("EncodingSymbol::from_payload: Raptor FEC encoding implementation is still in progress"); - throw "EncodingSymbol::from_payload Raptor FEC is not done yet"; - break; default: throw "Invalid FEC encoding ID. Only 2 FEC types are currently supported: compact no-code or raptor"; break; diff --git a/src/File.cpp b/src/File.cpp index 6bf2efa..f332dfd 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -94,7 +94,7 @@ LibFlute::File::File(uint32_t toi, break; case FecScheme::Raptor: _meta.fec_oti.transfer_length = length; - _meta.fec_transformer = new RaptorFEC(length, fec_oti.encoding_symbol_length, 1024*1024); // corresponding delete in FileDeliveryTable.cpp:remove() + _meta.fec_transformer = new RaptorFEC(length, fec_oti.encoding_symbol_length, 256*1024); // corresponding delete in FileDeliveryTable.cpp:remove() // set W (target on the sub block size, to be fairly large - 1 MB for now) spdlog::warn("File.cpp - Raptor FEC scheme is not done yet"); From f759caf4bf0ca1b76adb384e203a5c37d3f01d97 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Thu, 5 Jan 2023 09:12:27 +0100 Subject: [PATCH 28/61] Added initial encoder implementation --- CMakeLists.txt | 10 +++++----- include/RaptorFEC.h | 6 ++++++ include/flute_types.h | 20 ++++++++++++++++---- raptor | 2 +- src/CompactNoCode.cpp | 2 +- src/RaptorFEC.cpp | 40 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 include/RaptorFEC.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 94495b2..63669f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,15 +59,15 @@ target_sources(flute src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp src/RaptorFEC.cpp src/CompactNoCode.cpp PUBLIC - include/Receiver.h include/Transmitter.h include/File.h + include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h ) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/) -#if(ENABLE_RAPTOR) -# target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/raptor/) -# target_link_libraries(flute LINK_PUBLIC raptor) -#endif() +if(ENABLE_RAPTOR) + target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/raptor/) + target_link_libraries(flute LINK_PUBLIC raptor) +endif() target_link_libraries( flute LINK_PUBLIC diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h new file mode 100644 index 0000000..4860101 --- /dev/null +++ b/include/RaptorFEC.h @@ -0,0 +1,6 @@ +#ifndef LIBFLUTE_RAPTORFEC_H +#define LIBFLUTE_RAPTORFEC_H + +#include "raptor.h" + +#endif //LIBFLUTE_RAPTORFEC_H diff --git a/include/flute_types.h b/include/flute_types.h index ec350eb..e159be5 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -17,6 +17,8 @@ #include #include #include "tinyxml2.h" +#include "raptor.h" + #pragma once /** \mainpage LibFlute - ALC/FLUTE library * @@ -83,13 +85,13 @@ namespace LibFlute { virtual bool check_source_block_completion(SourceBlock& srcblk) = 0; /** - * @brief Encode a source block into multiple symbols + * @brief Encode a file into multiple source blocks * * @param buffer a pointer to the buffer containing the data * @param bytes_read a pointer to an integer to store the number of bytes read out of buffer * @return a map of source blocks that the object has been encoded to */ - virtual std::map create_blocks(char *buffer, int *bytes_read) = 0; + virtual std::map create_blocks(unsigned char *buffer, int *bytes_read) = 0; virtual bool calculate_partitioning() = 0; @@ -117,7 +119,7 @@ namespace LibFlute { bool check_source_block_completion(SourceBlock& srcblk); - std::map create_blocks(char *buffer, int *bytes_read); + std::map create_blocks(unsigned char *buffer, int *bytes_read); bool calculate_partitioning(); @@ -134,6 +136,16 @@ namespace LibFlute { }; class RaptorFEC : public FecTransformer { + + private: + + unsigned int target_K(); + + Symbol translate_symbol(struct enc_context *encoder_ctx); + + LibFlute::SourceBlock create_block(unsigned char *buffer, int *bytes_read); + + const float surplus_packet_ratio = 1.5; public: @@ -145,7 +157,7 @@ namespace LibFlute { bool check_source_block_completion(SourceBlock& srcblk); - std::map create_blocks(char *buffer, int *bytes_read); + std::map create_blocks(unsigned char *buffer, int *bytes_read); bool calculate_partitioning(); diff --git a/raptor b/raptor index d38cf5e..77258fb 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit d38cf5e3325ceb875edcd35d1ff4299dc7939431 +Subproject commit 77258fbf1c5a50f4cce4fd4abbb96d5ef96c0f75 diff --git a/src/CompactNoCode.cpp b/src/CompactNoCode.cpp index afb584c..f81e3d9 100644 --- a/src/CompactNoCode.cpp +++ b/src/CompactNoCode.cpp @@ -19,7 +19,7 @@ bool LibFlute::CompactNoCodeFEC::check_source_block_completion(LibFlute::SourceB return true; } -std::map LibFlute::CompactNoCodeFEC::create_blocks(char *buffer, int *bytes_read) { +std::map LibFlute::CompactNoCodeFEC::create_blocks(unsigned char *buffer, int *bytes_read) { std::map m; return m; } diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 9bf8691..48536a0 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -52,10 +52,44 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s return true; } -std::map LibFlute::RaptorFEC::create_blocks(char *buffer, int *bytes_read) { +unsigned int LibFlute::RaptorFEC::target_K() { return K * surplus_packet_ratio; } + +LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx) { + struct Symbol symbol { new char[T], T }; + struct LT_packet *lt_packet = encode_LT_packet(encoder_ctx); + + memcpy(symbol.data, <_packet->syms, T); + + free_LT_packet(lt_packet); + return symbol; +} + +LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(unsigned char *buffer, int *bytes_read) { + struct SourceBlock source_block; + struct enc_context *encoder_ctx = create_encoder_context(buffer, K, T); + unsigned int symbols_to_read = target_K(); + + for(unsigned int symbol_id = 1; symbol_id < symbols_to_read; symbol_id++) { + source_block.symbols[symbol_id] = translate_symbol(encoder_ctx); + *bytes_read += T; + } + + free_encoder_context(encoder_ctx); + return source_block; +} + + +// TODO: Reformat to K and T +std::map LibFlute::RaptorFEC::create_blocks(unsigned char *buffer, int *bytes_read) { // TODO: encode buffer into a number of symbols - std::map m; - return m; + if(N != 1) + throw std::invalid_argument("Currently the encoding only supports 1 sub-block per block"); + std::map block_map; + + for(unsigned int src_blocks = 1; src_blocks < Z + 1; src_blocks++) + block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read); + + return block_map; } From 34df0b17fdb850a3e7364587ab4ad724f205bb49 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 5 Jan 2023 09:31:24 +0100 Subject: [PATCH 29/61] resolve merge conflict --- src/RaptorFEC.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 48536a0..aa38def 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -1,11 +1,9 @@ -#include "flute_types.h" -#include "spdlog/spdlog.h" +#include "RaptorFEC.h" #ifdef RAPTOR_ENABLED LibFlute::RaptorFEC::RaptorFEC(){} - LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_payload, unsigned long target_sub_block_size) : F(transfer_length) , P(max_payload) @@ -15,7 +13,10 @@ LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_pa calculate_partitioning(); } -LibFlute::RaptorFEC::~RaptorFEC() = default; +LibFlute::RaptorFEC::~RaptorFEC() { + free_decoder_context(dc); + free_encoder_context(sc); +} bool LibFlute::RaptorFEC::calculate_partitioning() { // TODO: print debug statements and test @@ -49,7 +50,21 @@ bool LibFlute::RaptorFEC::calculate_partitioning() { bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { // TODO: try to decode srcblk using the symbols it contains... - return true; + // sc needs to have: snum, psize, cnum, graph + // graph for precoding is problematic: generated based on some randomness + // solution 1: transfer some (ideally slim) representation of graph from sndr to rcvr + // solution 2: transfer random seed from sndr to rcvr reproduce randomness and create exact graph + if(!sc) + { + // TODO: create slim version of encoder_context: only snum, psize, cnum and graph + sc = create_encoder_context(NULL, Z, T); + } + if(!dc) dc = create_decoder_context(sc); + // TODO: implement transform_srcblk_to_lt (LibFlute::SourcBlock -> LT_packet) + struct LT_packet * pkt = transform_srcblk_to_lt(srcblk); + process_LT_packet(dc, pkt); + free_LT_packet(pkt); + return dc->finished; } unsigned int LibFlute::RaptorFEC::target_K() { return K * surplus_packet_ratio; } @@ -149,4 +164,4 @@ bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { return true; } -#endif \ No newline at end of file +#endif From ab19d8aa916e7defb247a9fd2284f9de5fe3cb6e Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 5 Jan 2023 09:42:20 +0100 Subject: [PATCH 30/61] fix merge conflict --- include/flute_types.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/flute_types.h b/include/flute_types.h index e159be5..a86de64 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -59,6 +59,7 @@ namespace LibFlute { }; struct SourceBlock { + uint16_t id = 0; bool complete = false; std::map symbols; }; @@ -171,7 +172,13 @@ namespace LibFlute { uint32_t large_source_block_length = 0; uint32_t small_source_block_length = 0; uint32_t nof_large_source_blocks = 0; + + private: + struct enc_context *sc = NULL; + struct dec_context *dc = NULL; + + unsigned int S; // seed for random generator unsigned int F; // object size in bytes unsigned int Al = 4; // symbol alignment: 4 unsigned int T; // symbol size in bytes @@ -180,7 +187,7 @@ namespace LibFlute { unsigned int N; // number of sub-blocks per source block unsigned int K; // number of symbols in a source block unsigned int P; // maximum payload size: e.g. 1436 for ipv4 over 802.3 - + }; }; From 605f14f871bdba2d057c5d48658be206825b74b9 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 5 Jan 2023 09:43:30 +0100 Subject: [PATCH 31/61] resolve merge conflict --- include/RaptorFEC.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h index 4860101..ee1ca6a 100644 --- a/include/RaptorFEC.h +++ b/include/RaptorFEC.h @@ -1,6 +1,10 @@ #ifndef LIBFLUTE_RAPTORFEC_H #define LIBFLUTE_RAPTORFEC_H -#include "raptor.h" +#include "gf2matrix.h" // freeRaptor lib +#include "raptor10.h" // freeRaptor lib +#include "raptor.h" // raptor lib +#include "flute_types.h" +#include "spdlog/spdlog.h" #endif //LIBFLUTE_RAPTORFEC_H From 04f24e2d465ef6d69deb2c9e4e5078fe73f680ff Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 5 Jan 2023 09:57:41 +0100 Subject: [PATCH 32/61] remove old freeRaptor header --- include/RaptorFEC.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h index ee1ca6a..ff257da 100644 --- a/include/RaptorFEC.h +++ b/include/RaptorFEC.h @@ -1,8 +1,6 @@ #ifndef LIBFLUTE_RAPTORFEC_H #define LIBFLUTE_RAPTORFEC_H -#include "gf2matrix.h" // freeRaptor lib -#include "raptor10.h" // freeRaptor lib #include "raptor.h" // raptor lib #include "flute_types.h" #include "spdlog/spdlog.h" From e15daa254f810143e36dfd7dafc70d04ef3f85a4 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Thu, 5 Jan 2023 10:21:22 +0100 Subject: [PATCH 33/61] cmake fix --- CMakeLists.txt | 2 +- src/RaptorFEC.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 63669f2..31d13dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,7 +44,7 @@ if(ENABLE_RAPTOR) add_library(raptor STATIC) target_sources(raptor PRIVATE - raptor/bipartite.c raptor/decoder.c raptor/encoder.c raptor/galois.c raptor/gaussian.c raptor/pivoting.c + raptor/bipartite.c raptor/decoder.c raptor/encoder.c raptor/galois.c raptor/gaussian.c raptor/pivoting.c raptor/random.c PUBLIC raptor/raptor.h ) diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index aa38def..59980cf 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -61,7 +61,7 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s } if(!dc) dc = create_decoder_context(sc); // TODO: implement transform_srcblk_to_lt (LibFlute::SourcBlock -> LT_packet) - struct LT_packet * pkt = transform_srcblk_to_lt(srcblk); + struct LT_packet * pkt; // = transform_srcblk_to_lt(srcblk); process_LT_packet(dc, pkt); free_LT_packet(pkt); return dc->finished; From 8accbbbba220ebd7d32be70d7ca5fae98549375b Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Thu, 5 Jan 2023 10:23:29 +0100 Subject: [PATCH 34/61] Check for nullptr in create_blocks --- src/RaptorFEC.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 48536a0..808c8cf 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -81,8 +81,9 @@ LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(unsigned char *buffer, i // TODO: Reformat to K and T std::map LibFlute::RaptorFEC::create_blocks(unsigned char *buffer, int *bytes_read) { - // TODO: encode buffer into a number of symbols - if(N != 1) + if(!bytes_read) + throw std::invalid_argument("bytes_read pointer shouldn't be null"); + if(N != 1) throw std::invalid_argument("Currently the encoding only supports 1 sub-block per block"); std::map block_map; From 088f3ba45cc8686eb0abfa67fcc2dccc19f7fe16 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Thu, 5 Jan 2023 10:24:03 +0100 Subject: [PATCH 35/61] Calculate bytes_read correctly --- src/RaptorFEC.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 808c8cf..fa8ade5 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -69,10 +69,10 @@ LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(unsigned char *buffer, i struct enc_context *encoder_ctx = create_encoder_context(buffer, K, T); unsigned int symbols_to_read = target_K(); - for(unsigned int symbol_id = 1; symbol_id < symbols_to_read; symbol_id++) { + for(unsigned int symbol_id = 1; symbol_id < symbols_to_read + 1; symbol_id++) { source_block.symbols[symbol_id] = translate_symbol(encoder_ctx); - *bytes_read += T; } + *bytes_read += K * T; free_encoder_context(encoder_ctx); return source_block; @@ -85,7 +85,9 @@ std::map LibFlute::RaptorFEC::create_blocks(uns throw std::invalid_argument("bytes_read pointer shouldn't be null"); if(N != 1) throw std::invalid_argument("Currently the encoding only supports 1 sub-block per block"); - std::map block_map; + + std::map block_map; + *bytes_read = 0; for(unsigned int src_blocks = 1; src_blocks < Z + 1; src_blocks++) block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read); From 27992b0208d24c593555e6cae5ea3ed0ae8b70b2 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Thu, 5 Jan 2023 10:25:16 +0100 Subject: [PATCH 36/61] Replace TODOs --- src/RaptorFEC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index fa8ade5..99e9365 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -55,6 +55,7 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s unsigned int LibFlute::RaptorFEC::target_K() { return K * surplus_packet_ratio; } LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx) { + // TODO: Delete in the File destructor (or anywhere where applicable) struct Symbol symbol { new char[T], T }; struct LT_packet *lt_packet = encode_LT_packet(encoder_ctx); @@ -79,7 +80,6 @@ LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(unsigned char *buffer, i } -// TODO: Reformat to K and T std::map LibFlute::RaptorFEC::create_blocks(unsigned char *buffer, int *bytes_read) { if(!bytes_read) throw std::invalid_argument("bytes_read pointer shouldn't be null"); From 14215153e894ed11aa28167c2599f1bfc39401db Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 5 Jan 2023 10:45:15 +0100 Subject: [PATCH 37/61] update submodule --- raptor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raptor b/raptor index 77258fb..29e9beb 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 77258fbf1c5a50f4cce4fd4abbb96d5ef96c0f75 +Subproject commit 29e9bebf60d031e1d530cc0189c66b94524ce0e1 From 23689a2a25ba20751dcbc9d722c45d8fa9fab5a5 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 5 Jan 2023 15:56:41 +0100 Subject: [PATCH 38/61] dont calculate raptor parameters in partitioning() function, hardcoded W = 16 MB to keep N = 1 --- include/flute_types.h | 15 ++++++++-- src/File.cpp | 7 +++-- src/RaptorFEC.cpp | 64 ++++++++++++++++++++++++++++--------------- 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index a86de64..b14f28f 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -48,7 +48,12 @@ namespace LibFlute { */ enum class FecScheme { CompactNoCode, - Raptor + Raptor, + Reed_Solomon_GF_2_m, + LDPC_Staircase_Codes, + LDPC_Triangle_Codes, + Reed_Solomon_GF_2_8, + RaptorQ }; struct Symbol { @@ -150,7 +155,7 @@ namespace LibFlute { public: - RaptorFEC(unsigned int transfer_length, unsigned int max_payload, unsigned long target_sub_block_size); + RaptorFEC(unsigned int transfer_length, unsigned int max_payload); RaptorFEC(); @@ -166,6 +171,9 @@ namespace LibFlute { bool add_fdt_info(tinyxml2::XMLElement *file); + std::map transformers; // en / de coders depending on if we are the Receiver or Transmitter + // std::map encoders; + // std::map decoders; uint32_t nof_source_symbols = 0; uint32_t nof_source_blocks = 0; @@ -182,7 +190,8 @@ namespace LibFlute { unsigned int F; // object size in bytes unsigned int Al = 4; // symbol alignment: 4 unsigned int T; // symbol size in bytes - unsigned long W = 256*1024; // target on sub block size (arbitrarily set to maximum recommended value of 256 kB) + unsigned long W = 16*1024*1024; // target on sub block size- set default to 16 MB, to keep the number of sub-blocks, N, = 1 (you probably only need W >= 11MB to achieve this, assuming an ethernet mtu of ~1500 bytes, but we round to the nearest power of 2) + unsigned int G; // number of symbols per packet unsigned int Z; // number of source blocks unsigned int N; // number of sub-blocks per source block unsigned int K; // number of symbols in a source block diff --git a/src/File.cpp b/src/File.cpp index f332dfd..31b6384 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -94,8 +94,7 @@ LibFlute::File::File(uint32_t toi, break; case FecScheme::Raptor: _meta.fec_oti.transfer_length = length; - _meta.fec_transformer = new RaptorFEC(length, fec_oti.encoding_symbol_length, 256*1024); // corresponding delete in FileDeliveryTable.cpp:remove() - // set W (target on the sub block size, to be fairly large - 1 MB for now) + _meta.fec_transformer = new RaptorFEC(length, fec_oti.encoding_symbol_length); spdlog::warn("File.cpp - Raptor FEC scheme is not done yet"); @@ -147,6 +146,10 @@ auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> voi auto LibFlute::File::check_source_block_completion( SourceBlock& block ) -> void { + if (_meta.fec_transformer) { + block.complete = _meta.fec_transformer->check_source_block_completion(block); + return; + } block.complete = std::all_of(block.symbols.begin(), block.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); } diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 099ca32..45d8d91 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -4,33 +4,23 @@ LibFlute::RaptorFEC::RaptorFEC(){} -LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_payload, unsigned long target_sub_block_size) +LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_payload) : F(transfer_length) , P(max_payload) - , W(target_sub_block_size) { - Al = 4; - calculate_partitioning(); -} - -LibFlute::RaptorFEC::~RaptorFEC() { - free_decoder_context(dc); - free_encoder_context(sc); -} + double g = fmin( fmin(ceil((double)P*1024/(double)F), (double)P/(double)Al), 10.0f); + spdlog::debug("double g = fmin( fmin(ceil((double)P*1024/F), (double)P/(double)Al), 10.0f"); + spdlog::debug("G = {} = min( ceil({}*1024/{}), {}/{}, 10.0f)",g,P,F,P,Al); + G = (unsigned int) g; -bool LibFlute::RaptorFEC::calculate_partitioning() { - // TODO: print debug statements and test - double G = fmin( fmin(ceil((double)P*1024/(double)F), (double)P/(double)Al), 10.0f); - spdlog::debug("double G = fmin( fmin(ceil((double)P*1024/F), (double)P/(double)Al), 10.0f"); - spdlog::debug("G = {} = min( ceil({}*1024/{}), {}/{}, 10.0f)",G,P,F,P,Al); - T = (unsigned int) floor((double)P/(double)(Al*G)) * Al; - spdlog::debug("T = (unsigned int) floor((double)P/(double)(Al*G)) * Al"); - spdlog::debug("T = {} = floor({}/({}*{})) * {}",T,P,Al,G,Al); + T = (unsigned int) floor((double)P/(double)(Al*g)) * Al; + spdlog::debug("T = (unsigned int) floor((double)P/(double)(Al*g)) * Al"); + spdlog::debug("T = {} = floor({}/({}*{})) * {}",T,P,Al,g,Al); assert(T % Al == 0); // Symbol size T should be a multiple of symbol alignment parameter Al - double Kt = ceil((double)F/(double)T); + double Kt = ceil((double)F/(double)T); // total symbols spdlog::debug("double Kt = ceil((double)F/(double)T)"); spdlog::debug("Kt = {} = ceil({}/{})",Kt,F,T); @@ -38,13 +28,36 @@ bool LibFlute::RaptorFEC::calculate_partitioning() { spdlog::debug("Z = (unsigned int) ceil(Kt/8192)"); spdlog::debug("Z = {} = ceil({}/8192)",Z,Kt); + K = (Kt > 8192) ? 8192 : (unsigned int) Kt; // symbols per source block + spdlog::debug("K = {}",K); N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al ); spdlog::warn("N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al )"); spdlog::warn("N = {} = min( ceil( ceil({}/{}) * {}/{} ) , {}/{} )",N,Kt,Z,T,W,T,Al); - //TODO set the values that the File class needs... + // Set the values that the File class may need: + nof_source_symbols = (unsigned int) Kt; + nof_source_blocks = Z; + + small_source_block_length = (Z * K - nof_source_symbols) * T; // = (number of symbols in the final (small) source block, if nof_source_symbols isnt cleanly divisible by Z * K ) * symbol size + + // open question as to how we define "large source blocks" because either none of the remaining "regular" blocks are large, or all of them are, since raptor has a fixed block size + + /* + nof_large_source_blocks = K - (small_source_block_length != 0); // if we define a "large" source block as a normal one then its just the nof "regular" source blocks minus the nof small ones (which is either one or zero) + large_source_block_length = K * T; + */ + nof_large_source_blocks = 0; //for now argue that there are no "large" blocks, only regular and small ones + large_source_block_length = 0; +} + +LibFlute::RaptorFEC::~RaptorFEC() { + free_decoder_context(dc); + free_encoder_context(sc); +} + +bool LibFlute::RaptorFEC::calculate_partitioning() { return true; } @@ -150,13 +163,20 @@ bool LibFlute::RaptorFEC::parse_fdt_info(tinyxml2::XMLElement *file) { throw "Required field \"FEC-OTI-Symbol-Alignment-Parameter\" is missing for an object in the FDT"; } - //TODO: calculate other relevant values from these + // Set the values that are missing that we or the File class may need, follows the same logic as in calculate_partitioning() + nof_source_symbols = ceil((double)F / (double)T); + K = (nof_source_symbols > 8192) ? 8192 : nof_source_symbols; + + nof_source_blocks = Z; + small_source_block_length = (Z * K - nof_source_symbols) * T; + nof_large_source_blocks = 0; + large_source_block_length = 0; return true; } bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { - //TODO: do we need transfer length too? Does it change based on FecScheme? + //TODO: do we need to set transfer length too? I already gets set earlier. Does it change based on FecScheme? file->SetAttribute("FEC-OTI-FEC-Encoding-ID", (unsigned) FecScheme::Raptor); file->SetAttribute("FEC-OTI-Encoding-Symbol-Length", T); file->SetAttribute("FEC-OTI-Symbol-Alignment-Parameter", Al); From 39a6b643310647862d7ee94d2892cbf0940597f1 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 5 Jan 2023 18:10:51 +0100 Subject: [PATCH 39/61] finished raptor encoding + sending, removed compact no code subclass and file, cleaned up cmakelistst for raptor build case --- CMakeLists.txt | 14 ++++++--- include/flute_types.h | 68 +++++++++++++++++------------------------- src/CompactNoCode.cpp | 39 ------------------------ src/EncodingSymbol.cpp | 12 ++------ src/File.cpp | 33 ++++++++++++++------ src/RaptorFEC.cpp | 24 ++++++++------- src/Transmitter.cpp | 5 ++-- 7 files changed, 78 insertions(+), 117 deletions(-) delete mode 100644 src/CompactNoCode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 31d13dc..5431735 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,12 +40,11 @@ set(CMAKE_CXX_CLANG_TIDY clang-tidy) if(ENABLE_RAPTOR) message(STATUS "Compiling raptor library for Raptor FEC support. To disable support build with -DENABLE_RAPTOR=OFF") - add_compile_definitions(RAPTOR_ENABLED) add_library(raptor STATIC) target_sources(raptor PRIVATE raptor/bipartite.c raptor/decoder.c raptor/encoder.c raptor/galois.c raptor/gaussian.c raptor/pivoting.c raptor/random.c - PUBLIC + PUBLIC raptor/raptor.h ) target_link_libraries(raptor LINK_PUBLIC m) @@ -56,16 +55,23 @@ endif() add_library(flute "") target_sources(flute PRIVATE - src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp src/RaptorFEC.cpp src/CompactNoCode.cpp + src/Receiver.cpp src/Transmitter.cpp src/AlcPacket.cpp src/File.cpp src/EncodingSymbol.cpp src/FileDeliveryTable.cpp src/IpSec.cpp utils/base64.cpp PUBLIC - include/Receiver.h include/Transmitter.h include/File.h include/RaptorFEC.h + include/Receiver.h include/Transmitter.h include/File.h ) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/) if(ENABLE_RAPTOR) + add_compile_definitions(RAPTOR_ENABLED) target_include_directories(flute PUBLIC ${CMAKE_CURRENT_LIST_DIR}/raptor/) + target_sources(flute + PRIVATE + src/RaptorFEC.cpp + PUBLIC + include/RaptorFEC.h + ) target_link_libraries(flute LINK_PUBLIC raptor) endif() diff --git a/include/flute_types.h b/include/flute_types.h index b14f28f..84a4995 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -17,7 +17,9 @@ #include #include #include "tinyxml2.h" +#ifdef RAPTOR_ENABLED #include "raptor.h" +#endif #pragma once /** \mainpage LibFlute - ALC/FLUTE library @@ -97,13 +99,23 @@ namespace LibFlute { * @param bytes_read a pointer to an integer to store the number of bytes read out of buffer * @return a map of source blocks that the object has been encoded to */ - virtual std::map create_blocks(unsigned char *buffer, int *bytes_read) = 0; + virtual std::map create_blocks(char *buffer, int *bytes_read) = 0; virtual bool calculate_partitioning() = 0; + /** + * @brief Attempt to parse relevent information for decoding from the FDT + * + * @return success status + */ virtual bool parse_fdt_info(tinyxml2::XMLElement *file) = 0; + /** + * @brief Add relevant information about the FEC Scheme which the decoder may need, to the FDT + * + * @return success status + */ virtual bool add_fdt_info(tinyxml2::XMLElement *file) = 0; uint32_t nof_source_symbols = 0; @@ -112,58 +124,31 @@ namespace LibFlute { uint32_t small_source_block_length = 0; uint32_t nof_large_source_blocks = 0; - - }; - - class CompactNoCodeFEC : public FecTransformer { - - public: - - CompactNoCodeFEC(); - - virtual ~CompactNoCodeFEC(); - - bool check_source_block_completion(SourceBlock& srcblk); - - std::map create_blocks(unsigned char *buffer, int *bytes_read); - - bool calculate_partitioning(); - - bool parse_fdt_info(tinyxml2::XMLElement *file); - - bool add_fdt_info(tinyxml2::XMLElement *file); - - uint32_t nof_source_symbols = 0; - uint32_t nof_source_blocks = 0; - uint32_t large_source_block_length = 0; - uint32_t small_source_block_length = 0; - uint32_t nof_large_source_blocks = 0; - }; class RaptorFEC : public FecTransformer { - private: + private: - unsigned int target_K(); + unsigned int target_K(); - Symbol translate_symbol(struct enc_context *encoder_ctx); + Symbol translate_symbol(struct enc_context *encoder_ctx); - LibFlute::SourceBlock create_block(unsigned char *buffer, int *bytes_read); + LibFlute::SourceBlock create_block(char *buffer, int *bytes_read); - const float surplus_packet_ratio = 1.5; + const float surplus_packet_ratio = 1.5; public: RaptorFEC(unsigned int transfer_length, unsigned int max_payload); - RaptorFEC(); + RaptorFEC() {}; virtual ~RaptorFEC(); bool check_source_block_completion(SourceBlock& srcblk); - std::map create_blocks(unsigned char *buffer, int *bytes_read); + std::map create_blocks(char *buffer, int *bytes_read); bool calculate_partitioning(); @@ -181,12 +166,6 @@ namespace LibFlute { uint32_t small_source_block_length = 0; uint32_t nof_large_source_blocks = 0; - private: - - struct enc_context *sc = NULL; - struct dec_context *dc = NULL; - - unsigned int S; // seed for random generator unsigned int F; // object size in bytes unsigned int Al = 4; // symbol alignment: 4 unsigned int T; // symbol size in bytes @@ -197,6 +176,13 @@ namespace LibFlute { unsigned int K; // number of symbols in a source block unsigned int P; // maximum payload size: e.g. 1436 for ipv4 over 802.3 + private: + +#ifdef RAPTOR_ENABLED + struct enc_context *sc = NULL; + struct dec_context *dc = NULL; +#endif + unsigned int S; // seed for random generator }; }; diff --git a/src/CompactNoCode.cpp b/src/CompactNoCode.cpp deleted file mode 100644 index f81e3d9..0000000 --- a/src/CompactNoCode.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "flute_types.h" -#include "spdlog/spdlog.h" - -LibFlute::CompactNoCodeFEC::CompactNoCodeFEC() { -// this->encoding_id = LibFlute::FecScheme::CompactNoCode; -} - -// LibFlute::CompactNoCodeFEC::CompactNoCodeFEC( uint64_t transfer_length, uint32_t encoding_symbol_length, uint32_t max_source_block_length) -// { -// this->encoding_symbol_length = encoding_symbol_length; -// this->transfer_length = transfer_length; -// this->max_source_block_length = max_source_block_length; -// this->encoding_id = LibFlute::FecScheme::CompactNoCode; -// } - -LibFlute::CompactNoCodeFEC::~CompactNoCodeFEC() = default; - -bool LibFlute::CompactNoCodeFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk){ - return true; -} - -std::map LibFlute::CompactNoCodeFEC::create_blocks(unsigned char *buffer, int *bytes_read) { - std::map m; - return m; -} - -bool LibFlute::CompactNoCodeFEC::calculate_partitioning(){ - return false; -} - -bool LibFlute::CompactNoCodeFEC::parse_fdt_info(tinyxml2::XMLElement *file) { - // TODO - return true; -} - -bool LibFlute::CompactNoCodeFEC::add_fdt_info(tinyxml2::XMLElement *file) { - // TODO - return true; -} \ No newline at end of file diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index 6175dd3..c18af54 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -70,17 +70,13 @@ auto LibFlute::EncodingSymbol::to_payload(const std::vector& sym auto first_symbol = symbols.begin(); switch (fec_oti.encoding_id) { case FecScheme::CompactNoCode: + case FecScheme::Raptor: *((uint16_t*)ptr) = htons(first_symbol->source_block_number()); ptr += 2; *((uint16_t*)ptr) = htons(first_symbol->id()); ptr += 2; len += 4; break; - case FecScheme::Raptor: - //TODO - spdlog::warn("EncodingSymbol::to_payload: Raptor FEC encoding implementation is still in progress"); - throw "EncodingSymbol::to_payload Raptor FEC is not done yet"; - break; default: throw "Invalid FEC encoding ID. Only 2 FEC types are currently supported: compact no-code or raptor"; break; @@ -115,16 +111,12 @@ auto LibFlute::EncodingSymbol::decode_to(char* buffer, size_t max_length) const auto LibFlute::EncodingSymbol::encode_to(char* buffer, size_t max_length) const -> size_t { switch (_fec_scheme) { case FecScheme::CompactNoCode: + case FecScheme::Raptor: if (_data_len <= max_length) { memcpy(buffer, _encoded_data, _data_len); return _data_len; } break; - case FecScheme::Raptor: - //TODO - spdlog::warn("EncodingSymbol::encode_to Raptor FEC encoding implementation is still in progress"); - throw "EncodingSymbol::encode_to Raptor FEC is not done yet"; - break; } return 0; } diff --git a/src/File.cpp b/src/File.cpp index 31b6384..217b4a0 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -87,20 +87,24 @@ LibFlute::File::File(uint32_t toi, _meta.expires = expires; _meta.fec_oti = fec_oti; +#ifdef RAPTOR_ENABLED + RaptorFEC *r = nullptr; +#endif + switch (_meta.fec_oti.encoding_id) { case FecScheme::CompactNoCode: _meta.fec_oti.transfer_length = length; _meta.fec_transformer = 0; break; +#ifdef RAPTOR_ENABLED case FecScheme::Raptor: _meta.fec_oti.transfer_length = length; - _meta.fec_transformer = new RaptorFEC(length, fec_oti.encoding_symbol_length); - - spdlog::warn("File.cpp - Raptor FEC scheme is not done yet"); - - // TODO... what else is missing here? - + r = new RaptorFEC(length, fec_oti.encoding_symbol_length); + _meta.fec_oti.encoding_symbol_length = r->T; + _meta.fec_oti.max_source_block_length = r->K * r->T; + _meta.fec_transformer = r; break; +#endif default: throw "FEC scheme not supported or not yet implemented"; break; @@ -180,7 +184,7 @@ auto LibFlute::File::check_file_completion() -> void auto LibFlute::File::calculate_partitioning() -> void { - if (0 && _meta.fec_transformer && _meta.fec_transformer->calculate_partitioning()){ + if (_meta.fec_transformer && _meta.fec_transformer->calculate_partitioning()){ _nof_source_symbols = _meta.fec_transformer->nof_source_symbols; _nof_source_blocks = _meta.fec_transformer->nof_source_blocks; _large_source_block_length = _meta.fec_transformer->large_source_block_length; @@ -199,6 +203,17 @@ auto LibFlute::File::calculate_partitioning() -> void auto LibFlute::File::create_blocks() -> void { // Create the required source blocks and encoding symbols + + if (_meta.fec_transformer){ + int bytes_read = 0; + _source_blocks = _meta.fec_transformer->create_blocks(_buffer, &bytes_read); + if (bytes_read < length()) { + spdlog::error("FEC Transformer failed to create source blocks"); + throw "FEC Transformer failed to create source blocks"; + } + return; + } + auto buffer_ptr = _buffer; size_t remaining_size = _meta.fec_oti.transfer_length; auto number = 0; @@ -226,10 +241,10 @@ auto LibFlute::File::create_blocks() -> void auto LibFlute::File::get_next_symbols(size_t max_size) -> std::vector { auto block = _source_blocks.begin(); - int nof_symbols = std::ceil((float)(max_size - 4) / (float)_meta.fec_oti.encoding_symbol_length); + int nof_symbols = std::floor((float)max_size / (float)_meta.fec_oti.encoding_symbol_length); auto cnt = 0; std::vector symbols; - + spdlog::debug("Attempting to queue {} symbols",nof_symbols); for (auto& block : _source_blocks) { if (cnt >= nof_symbols) break; diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 45d8d91..3dfab1b 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -1,9 +1,5 @@ #include "RaptorFEC.h" -#ifdef RAPTOR_ENABLED - -LibFlute::RaptorFEC::RaptorFEC(){} - LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_payload) : F(transfer_length) , P(max_payload) @@ -62,6 +58,10 @@ bool LibFlute::RaptorFEC::calculate_partitioning() { } bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { + if (!dc || !transformers.size()) { + // check source block completion for the ENcoder + return std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); + } // TODO: try to decode srcblk using the symbols it contains... // sc needs to have: snum, psize, cnum, graph // graph for precoding is problematic: generated based on some randomness @@ -93,22 +93,22 @@ LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encod return symbol; } -LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(unsigned char *buffer, int *bytes_read) { +LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(char *buffer, int *bytes_read) { struct SourceBlock source_block; - struct enc_context *encoder_ctx = create_encoder_context(buffer, K, T); + struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T); + *bytes_read += K * T; unsigned int symbols_to_read = target_K(); for(unsigned int symbol_id = 1; symbol_id < symbols_to_read + 1; symbol_id++) { source_block.symbols[symbol_id] = translate_symbol(encoder_ctx); } - *bytes_read += K * T; free_encoder_context(encoder_ctx); return source_block; } -std::map LibFlute::RaptorFEC::create_blocks(unsigned char *buffer, int *bytes_read) { +std::map LibFlute::RaptorFEC::create_blocks(char *buffer, int *bytes_read) { if(!bytes_read) throw std::invalid_argument("bytes_read pointer shouldn't be null"); if(N != 1) @@ -163,6 +163,10 @@ bool LibFlute::RaptorFEC::parse_fdt_info(tinyxml2::XMLElement *file) { throw "Required field \"FEC-OTI-Symbol-Alignment-Parameter\" is missing for an object in the FDT"; } + if (T % Al) { + throw "Symbol size T is not a multiple of Al. Invalid configuration from sender"; + } + // Set the values that are missing that we or the File class may need, follows the same logic as in calculate_partitioning() nof_source_symbols = ceil((double)F / (double)T); K = (nof_source_symbols > 8192) ? 8192 : nof_source_symbols; @@ -185,6 +189,4 @@ bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { file->SetAttribute("FEC-OTI-Symbol-Alignment-Parameter", Al); return true; -} - -#endif +} \ No newline at end of file diff --git a/src/Transmitter.cpp b/src/Transmitter.cpp index d676133..bca3cb7 100644 --- a/src/Transmitter.cpp +++ b/src/Transmitter.cpp @@ -36,10 +36,10 @@ boost::asio::io_service& io_service) ,_fec_scheme(fec_scheme) { _max_payload = mtu - - 20 - // IPv4 header + ( _endpoint.address().is_v6() ? 40 : 20) - // IP header 8 - // UDP header 32 - // ALC Header with EXT_FDT and EXT_FTI - 4; // SBN and ESI for compact no-code FEC + 4; // SBN and ESI for compact no-code or raptor FEC uint32_t max_source_block_length = 64; switch(_fec_scheme) { @@ -55,7 +55,6 @@ boost::asio::io_service& io_service) _socket.set_option(boost::asio::ip::multicast::enable_loopback(true)); _socket.set_option(boost::asio::ip::udp::socket::reuse_address(true)); - //TODO: calculate max payload and max source block length properly based on FEC scheme _fec_oti = FecOti{_fec_scheme, 0, _max_payload, max_source_block_length}; _fdt = std::make_unique(1, _fec_oti); From 6851b440eaeffa582e528b51851a641ee958b724 Mon Sep 17 00:00:00 2001 From: SteezyE Date: Sat, 7 Jan 2023 00:13:15 +0100 Subject: [PATCH 40/61] integrated raptor decoding functionality with libflute --- include/RaptorFEC.h | 1 + include/flute_types.h | 21 ++++++++------ raptor | 2 +- src/RaptorFEC.cpp | 66 ++++++++++++++++++++++++++++--------------- 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/include/RaptorFEC.h b/include/RaptorFEC.h index ff257da..8be22eb 100644 --- a/include/RaptorFEC.h +++ b/include/RaptorFEC.h @@ -4,5 +4,6 @@ #include "raptor.h" // raptor lib #include "flute_types.h" #include "spdlog/spdlog.h" +#include #endif //LIBFLUTE_RAPTORFEC_H diff --git a/include/flute_types.h b/include/flute_types.h index 84a4995..657720f 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -61,12 +61,19 @@ namespace LibFlute { struct Symbol { char* data; size_t length; +#ifdef RAPTOR_ENABLED + int* sid; + int deg; +#endif bool complete = false; bool queued = false; }; struct SourceBlock { uint16_t id = 0; +#ifdef RAPTOR_ENABLED + uint32_t seed; +#endif bool complete = false; std::map symbols; }; @@ -137,6 +144,11 @@ namespace LibFlute { LibFlute::SourceBlock create_block(char *buffer, int *bytes_read); const float surplus_packet_ratio = 1.5; + +#ifdef RAPTOR_ENABLED + struct enc_context *sc = NULL; + struct dec_context *dc = NULL; +#endif public: @@ -165,7 +177,7 @@ namespace LibFlute { uint32_t large_source_block_length = 0; uint32_t small_source_block_length = 0; uint32_t nof_large_source_blocks = 0; - + unsigned int F; // object size in bytes unsigned int Al = 4; // symbol alignment: 4 unsigned int T; // symbol size in bytes @@ -175,14 +187,7 @@ namespace LibFlute { unsigned int N; // number of sub-blocks per source block unsigned int K; // number of symbols in a source block unsigned int P; // maximum payload size: e.g. 1436 for ipv4 over 802.3 - - private: -#ifdef RAPTOR_ENABLED - struct enc_context *sc = NULL; - struct dec_context *dc = NULL; -#endif - unsigned int S; // seed for random generator }; }; diff --git a/raptor b/raptor index 29e9beb..a7131ec 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 29e9bebf60d031e1d530cc0189c66b94524ce0e1 +Subproject commit a7131ecccb96f5b6caf0b62e4927460de5e036c8 diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 3dfab1b..84b7329 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -9,7 +9,6 @@ LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_pa spdlog::debug("G = {} = min( ceil({}*1024/{}), {}/{}, 10.0f)",g,P,F,P,Al); G = (unsigned int) g; - T = (unsigned int) floor((double)P/(double)(Al*g)) * Al; spdlog::debug("T = (unsigned int) floor((double)P/(double)(Al*g)) * Al"); spdlog::debug("T = {} = floor({}/({}*{})) * {}",T,P,Al,g,Al); @@ -58,36 +57,59 @@ bool LibFlute::RaptorFEC::calculate_partitioning() { } bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { +// TODO: discuss / realize transformers idea +#if 0 if (!dc || !transformers.size()) { - // check source block completion for the ENcoder + // check source block completion for the Encoder return std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); } - // TODO: try to decode srcblk using the symbols it contains... - // sc needs to have: snum, psize, cnum, graph - // graph for precoding is problematic: generated based on some randomness - // solution 1: transfer some (ideally slim) representation of graph from sndr to rcvr - // solution 2: transfer random seed from sndr to rcvr reproduce randomness and create exact graph - if(!sc) +#endif + if(!srcblk.symbols.size()) + return false; + if(sc) + free_encoder_context(sc); + if(dc) + free_decoder_context(dc); + + sc = create_encoder_context(NULL, K, T, srcblk.seed); + dc = create_decoder_context(sc); + + // std::map::iterator + for(auto iter = srcblk.symbols.begin(); iter != srcblk.symbols.end(); iter++) { - // TODO: create slim version of encoder_context: only snum, psize, cnum and graph - sc = create_encoder_context(NULL, Z, T); + /** struct LT_packet { + * int id; // packet id (number of packet e.g. id=2 <=> 2. packet) + * int deg; // packet degree (number of xored symbols) + * int *sid; // id of source packets in the packet + * GF_ELEMENT *syms; // symbols of coded packet + * }; + * + * symbol.data <=> LT_packet.syms + * symbol.deg <=> LT_packet.deg + * symbol.sid <=> LT_packet.sid + */ + struct LT_packet * pkt = (struct LT_packet *) calloc(1, sizeof(struct LT_packet)); + pkt->deg = iter->second.deg; + pkt->sid = (int *) calloc(pkt->deg, sizeof(int)); + pkt->syms = (GF_ELEMENT *) calloc(iter->second.length, sizeof(char)); + memcpy(pkt->sid, iter->second.sid, pkt->deg * sizeof(int)); + memcpy(pkt->syms, iter->second.data, pkt->deg * sizeof(char)); + + process_LT_packet(dc, pkt); + free_LT_packet(pkt); } - if(!dc) dc = create_decoder_context(sc); - // TODO: implement transform_srcblk_to_lt (LibFlute::SourcBlock -> LT_packet) - struct LT_packet * pkt; // = transform_srcblk_to_lt(srcblk); - process_LT_packet(dc, pkt); - free_LT_packet(pkt); return dc->finished; } unsigned int LibFlute::RaptorFEC::target_K() { return K * surplus_packet_ratio; } -LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx) { +LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx){ // TODO: Delete in the File destructor (or anywhere where applicable) - struct Symbol symbol { new char[T], T }; struct LT_packet *lt_packet = encode_LT_packet(encoder_ctx); + struct Symbol symbol { new char[T], T, new int[lt_packet->deg], lt_packet->deg }; - memcpy(symbol.data, <_packet->syms, T); + memcpy(symbol.data, lt_packet->syms, T); + memcpy(symbol.sid, lt_packet->sid, lt_packet->deg * sizeof(int)); free_LT_packet(lt_packet); return symbol; @@ -95,15 +117,15 @@ LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encod LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(char *buffer, int *bytes_read) { struct SourceBlock source_block; - struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T); - *bytes_read += K * T; + source_block.seed = (uint32_t) rand(); + struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T, source_block.seed); unsigned int symbols_to_read = target_K(); for(unsigned int symbol_id = 1; symbol_id < symbols_to_read + 1; symbol_id++) { source_block.symbols[symbol_id] = translate_symbol(encoder_ctx); } + *bytes_read += K * T; - free_encoder_context(encoder_ctx); return source_block; } @@ -189,4 +211,4 @@ bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { file->SetAttribute("FEC-OTI-Symbol-Alignment-Parameter", Al); return true; -} \ No newline at end of file +} From 3087ddd1733d754bb98891db3ddab45c0ccc61bb Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 12 Jan 2023 12:26:06 +0100 Subject: [PATCH 41/61] finish decoder, start SBN and ESI at 0, added process_symbol() function for all FEC transformers --- include/flute_types.h | 33 +++++++------ raptor | 2 +- src/File.cpp | 5 +- src/RaptorFEC.cpp | 111 ++++++++++++++++++++++-------------------- 4 files changed, 80 insertions(+), 71 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 657720f..0c486b7 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -61,19 +61,12 @@ namespace LibFlute { struct Symbol { char* data; size_t length; -#ifdef RAPTOR_ENABLED - int* sid; - int deg; -#endif bool complete = false; bool queued = false; }; struct SourceBlock { uint16_t id = 0; -#ifdef RAPTOR_ENABLED - uint32_t seed; -#endif bool complete = false; std::map symbols; }; @@ -108,6 +101,15 @@ namespace LibFlute { */ virtual std::map create_blocks(char *buffer, int *bytes_read) = 0; + /** + * @brief Process a received symbol + * + * @param srcblk the source block this symbols corresponds to + * @param symb the received symbol + * @param id the symbols id + * @return success or failure + */ + virtual bool process_symbol(LibFlute::SourceBlock& srcblk, LibFlute::Symbol& symb, unsigned int id) = 0; virtual bool calculate_partitioning() = 0; @@ -141,15 +143,12 @@ namespace LibFlute { Symbol translate_symbol(struct enc_context *encoder_ctx); - LibFlute::SourceBlock create_block(char *buffer, int *bytes_read); + LibFlute::SourceBlock create_block(char *buffer, int *bytes_read, int blockid); const float surplus_packet_ratio = 1.5; - -#ifdef RAPTOR_ENABLED - struct enc_context *sc = NULL; - struct dec_context *dc = NULL; -#endif + void extract_finished_block(LibFlute::SourceBlock& srcblk, struct dec_context *dc); + public: RaptorFEC(unsigned int transfer_length, unsigned int max_payload); @@ -162,15 +161,17 @@ namespace LibFlute { std::map create_blocks(char *buffer, int *bytes_read); + bool process_symbol(LibFlute::SourceBlock& srcblk, LibFlute::Symbol& symb, unsigned int id); + bool calculate_partitioning(); bool parse_fdt_info(tinyxml2::XMLElement *file); bool add_fdt_info(tinyxml2::XMLElement *file); - std::map transformers; // en / de coders depending on if we are the Receiver or Transmitter - // std::map encoders; - // std::map decoders; +#ifdef RAPTOR_ENABLED + std::map decoders; // map of source block number to decoders +#endif uint32_t nof_source_symbols = 0; uint32_t nof_source_blocks = 0; diff --git a/raptor b/raptor index a7131ec..0570760 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit a7131ecccb96f5b6caf0b62e4927460de5e036c8 +Subproject commit 0570760bf91d8050d5229e5fcb674c38f0b400a6 diff --git a/src/File.cpp b/src/File.cpp index 217b4a0..34a2584 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -141,7 +141,10 @@ auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> voi if (!target_symbol.complete) { symbol.decode_to(target_symbol.data, target_symbol.length); target_symbol.complete = true; - + if (_meta.fec_transformer) { + _meta.fec_transformer->process_symbol(source_block,target_symbol,symbol.id()); + return; + } check_source_block_completion(source_block); check_file_completion(); } diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 84b7329..2b71951 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -48,55 +48,58 @@ LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_pa } LibFlute::RaptorFEC::~RaptorFEC() { - free_decoder_context(dc); - free_encoder_context(sc); + for(auto iter = decoders.begin(); iter != decoders.end(); iter++){ + free_decoder_context(iter->second); + } } bool LibFlute::RaptorFEC::calculate_partitioning() { return true; } +void LibFlute::RaptorFEC::extract_finished_block(LibFlute::SourceBlock& srcblk, struct dec_context *dc) { + for(auto iter = srcblk.symbols.begin(); iter != srcblk.symbols.end(); iter++) { + memcpy(iter->second.data,dc->pp[iter->first],T); // overwrite the encoded symbol with the source data; + } +} + +bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute::Symbol& symbol, unsigned int id) { + assert(symbol.length == T); // symbol.length should always be T (the symbols size) + struct dec_context *dc = decoders[srcblk.id]; + if (!dc) { + struct enc_context *sc = create_encoder_context(NULL, K, T, srcblk.id); + dc = create_decoder_context(sc); + decoders[srcblk.id] = dc; + } + struct LT_packet * pkt = (struct LT_packet *) calloc(1, sizeof(struct LT_packet)); + pkt->id = id; + pkt->syms = (GF_ELEMENT *) calloc(symbol.length, sizeof(char)); + memcpy(pkt->syms, symbol.data, symbol.length * sizeof(char)); + + process_LT_packet(dc, pkt); + free_LT_packet(pkt); + return true; +} + + bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { -// TODO: discuss / realize transformers idea -#if 0 - if (!dc || !transformers.size()) { + if (!decoders.size()) { // check source block completion for the Encoder return std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); } -#endif - if(!srcblk.symbols.size()) - return false; - if(sc) - free_encoder_context(sc); - if(dc) - free_decoder_context(dc); - - sc = create_encoder_context(NULL, K, T, srcblk.seed); - dc = create_decoder_context(sc); - - // std::map::iterator - for(auto iter = srcblk.symbols.begin(); iter != srcblk.symbols.end(); iter++) - { - /** struct LT_packet { - * int id; // packet id (number of packet e.g. id=2 <=> 2. packet) - * int deg; // packet degree (number of xored symbols) - * int *sid; // id of source packets in the packet - * GF_ELEMENT *syms; // symbols of coded packet - * }; - * - * symbol.data <=> LT_packet.syms - * symbol.deg <=> LT_packet.deg - * symbol.sid <=> LT_packet.sid - */ - struct LT_packet * pkt = (struct LT_packet *) calloc(1, sizeof(struct LT_packet)); - pkt->deg = iter->second.deg; - pkt->sid = (int *) calloc(pkt->deg, sizeof(int)); - pkt->syms = (GF_ELEMENT *) calloc(iter->second.length, sizeof(char)); - memcpy(pkt->sid, iter->second.sid, pkt->deg * sizeof(int)); - memcpy(pkt->syms, iter->second.data, pkt->deg * sizeof(char)); - - process_LT_packet(dc, pkt); - free_LT_packet(pkt); + // else case- we are the Decoder + if(!srcblk.symbols.size()){ + spdlog::warn("Empty source block (size 0) SBN {}",srcblk.id); + return false; + } + + struct dec_context *dc = decoders[srcblk.id]; + if (!dc) { + spdlog::error("Couldnt find raptor decoder for source block {}",srcblk.id); + return false; + } + if (dc->finished) { + extract_finished_block(srcblk,dc); } return dc->finished; } @@ -106,41 +109,43 @@ unsigned int LibFlute::RaptorFEC::target_K() { return K * surplus_packet_ratio; LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx){ // TODO: Delete in the File destructor (or anywhere where applicable) struct LT_packet *lt_packet = encode_LT_packet(encoder_ctx); - struct Symbol symbol { new char[T], T, new int[lt_packet->deg], lt_packet->deg }; + struct Symbol symbol { new char[T], T}; memcpy(symbol.data, lt_packet->syms, T); - memcpy(symbol.sid, lt_packet->sid, lt_packet->deg * sizeof(int)); free_LT_packet(lt_packet); return symbol; } -LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(char *buffer, int *bytes_read) { +LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(char *buffer, int *bytes_read, int blockid) { struct SourceBlock source_block; - source_block.seed = (uint32_t) rand(); - struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T, source_block.seed); + source_block.id = blockid; + int seed = blockid; + struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T, seed); unsigned int symbols_to_read = target_K(); - for(unsigned int symbol_id = 1; symbol_id < symbols_to_read + 1; symbol_id++) { + for(unsigned int symbol_id = 0; symbol_id < symbols_to_read; symbol_id++) { source_block.symbols[symbol_id] = translate_symbol(encoder_ctx); } *bytes_read += K * T; + free_encoder_context(encoder_ctx); return source_block; } std::map LibFlute::RaptorFEC::create_blocks(char *buffer, int *bytes_read) { - if(!bytes_read) - throw std::invalid_argument("bytes_read pointer shouldn't be null"); - if(N != 1) - throw std::invalid_argument("Currently the encoding only supports 1 sub-block per block"); + if(!bytes_read) + throw std::invalid_argument("bytes_read pointer shouldn't be null"); + if(N != 1) + throw std::invalid_argument("Currently the encoding only supports 1 sub-block per block"); - std::map block_map; - *bytes_read = 0; + std::map block_map; + *bytes_read = 0; - for(unsigned int src_blocks = 1; src_blocks < Z + 1; src_blocks++) - block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read); + for(unsigned int src_blocks = 0; src_blocks < Z; src_blocks++) { + block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read, src_blocks); + } return block_map; } From d1c7d7fcf799ee92f9f88997628703df021e9efa Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Mon, 16 Jan 2023 18:24:20 +0100 Subject: [PATCH 42/61] fixes for running receiver --- include/flute_types.h | 2 ++ src/AlcPacket.cpp | 14 ++++++++++---- src/EncodingSymbol.cpp | 13 +++++-------- src/File.cpp | 6 +++++- src/FileDeliveryTable.cpp | 1 + src/RaptorFEC.cpp | 29 ++++++++++++++++++++++++----- 6 files changed, 47 insertions(+), 18 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 0c486b7..600b999 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -139,6 +139,8 @@ namespace LibFlute { private: + bool is_encoder = true; + unsigned int target_K(); Symbol translate_symbol(struct enc_context *encoder_ctx); diff --git a/src/AlcPacket.cpp b/src/AlcPacket.cpp index 4741141..12e0002 100644 --- a/src/AlcPacket.cpp +++ b/src/AlcPacket.cpp @@ -80,10 +80,16 @@ LibFlute::AlcPacket::AlcPacket(char* data, size_t len) throw "TOI fields over 64 bits in length are not supported"; } - if (_lct_header.codepoint == 0) { - _fec_oti.encoding_id = FecScheme::CompactNoCode; - } else { - throw "Only Compact No-Code FEC is supported"; + switch (_lct_header.codepoint) { + case 0: + _fec_oti.encoding_id = FecScheme::CompactNoCode; + break; + case 1: + _fec_oti.encoding_id = FecScheme::Raptor; + break; + default: + throw "Only Compact No-Code FEC is supported"; + break; } auto expected_header_len = 2 + diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index c18af54..a0abd1e 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -49,11 +49,8 @@ auto LibFlute::EncodingSymbol::from_payload(char* encoded_data, size_t data_len, for (int i = 0; i < nof_symbols; i++) { switch(fec_oti.encoding_id) { case FecScheme::CompactNoCode: - symbols.emplace_back(encoding_symbol_id, source_block_number, encoded_data, std::min(data_len, (size_t)fec_oti.encoding_symbol_length), fec_oti.encoding_id); - break; case FecScheme::Raptor: - //TODO - throw "Raptor support is still on its way"; + symbols.emplace_back(encoding_symbol_id, source_block_number, encoded_data, std::min(data_len, (size_t)fec_oti.encoding_symbol_length), fec_oti.encoding_id); break; } encoded_data += fec_oti.encoding_symbol_length; @@ -96,14 +93,14 @@ auto LibFlute::EncodingSymbol::to_payload(const std::vector& sym auto LibFlute::EncodingSymbol::decode_to(char* buffer, size_t max_length) const -> void { switch (_fec_scheme) { case FecScheme::CompactNoCode: + case FecScheme::Raptor: if (_data_len <= max_length) { memcpy(buffer, _encoded_data, _data_len); } break; - case FecScheme::Raptor: - //TODO - spdlog::warn("EncodingSymbol::decode_to Raptor FEC encoding implementation is still in progress"); - throw "EncodingSymbol::decode_to Raptor FEC is not done yet"; + default: + spdlog::warn("EncodingSymbol::decode_to() called for unknown fec scheme {}",(int)_fec_scheme); + throw "EncodingSymbol::decode_to() called for unknown fec scheme"; break; } } diff --git a/src/File.cpp b/src/File.cpp index 34a2584..f2418aa 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -126,6 +126,10 @@ LibFlute::File::~File() auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> void { + if(_complete) { + spdlog::debug("Skipped processing symbol {} , SBN {} since file is already complete",symbol.id(),symbol.source_block_number()); + return; + } if (symbol.source_block_number() > _source_blocks.size()) { throw "Source Block number too high"; } @@ -210,7 +214,7 @@ auto LibFlute::File::create_blocks() -> void if (_meta.fec_transformer){ int bytes_read = 0; _source_blocks = _meta.fec_transformer->create_blocks(_buffer, &bytes_read); - if (bytes_read < length()) { + if (_source_blocks.size() <= 0) { spdlog::error("FEC Transformer failed to create source blocks"); throw "FEC Transformer failed to create source blocks"; } diff --git a/src/FileDeliveryTable.cpp b/src/FileDeliveryTable.cpp index 029da4e..37c064c 100644 --- a/src/FileDeliveryTable.cpp +++ b/src/FileDeliveryTable.cpp @@ -126,6 +126,7 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffe switch (encoding_id){ case (int) FecScheme::Raptor: fec_transformer = new RaptorFEC(); // corresponding delete calls in Receiver.cpp and destuctor function + spdlog::debug("Received FDT entry for a raptor encoded file"); break; default: break; diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 2b71951..bd39c1a 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -64,13 +64,17 @@ void LibFlute::RaptorFEC::extract_finished_block(LibFlute::SourceBlock& srcblk, } bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute::Symbol& symbol, unsigned int id) { - assert(symbol.length == T); // symbol.length should always be T (the symbols size) + assert(symbol.length == T); // symbol.length should always be T (the symbol's size) struct dec_context *dc = decoders[srcblk.id]; if (!dc) { struct enc_context *sc = create_encoder_context(NULL, K, T, srcblk.id); dc = create_decoder_context(sc); decoders[srcblk.id] = dc; } + if (dc->finished){ + spdlog::warn("Skipped processing of symbol for finished block : SBN {}, ESI {}",srcblk.id,id); + return true; + } struct LT_packet * pkt = (struct LT_packet *) calloc(1, sizeof(struct LT_packet)); pkt->id = id; pkt->syms = (GF_ELEMENT *) calloc(symbol.length, sizeof(char)); @@ -83,11 +87,12 @@ bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { - if (!decoders.size()) { + if (is_encoder) { // check source block completion for the Encoder return std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); } // else case- we are the Decoder + if(!srcblk.symbols.size()){ spdlog::warn("Empty source block (size 0) SBN {}",srcblk.id); return false; @@ -98,6 +103,7 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s spdlog::error("Couldnt find raptor decoder for source block {}",srcblk.id); return false; } + spdlog::debug("raptor is finished? : {}",dc->finished); if (dc->finished) { extract_finished_block(srcblk,dc); } @@ -144,15 +150,26 @@ std::map LibFlute::RaptorFEC::create_blocks(cha *bytes_read = 0; for(unsigned int src_blocks = 0; src_blocks < Z; src_blocks++) { - block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read, src_blocks); + if(!is_encoder) { + LibFlute::SourceBlock block; + unsigned int symbols_to_read = target_K(); + for (int i = 0; i < symbols_to_read; i++) { + block.symbols[i] = Symbol {.data = buffer + T*i, .length = T, .complete = false}; + } + block_map[src_blocks] = block; + } else { + block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read, src_blocks); + } + } + if(!is_encoder) { + spdlog::debug("Raptor Decoder- prepared {} empty source blocks with {} symbols (K = {})",block_map.size(),target_K(),K); } - return block_map; } bool LibFlute::RaptorFEC::parse_fdt_info(tinyxml2::XMLElement *file) { - // TODO + is_encoder = false; const char* val = 0; val = file->Attribute("Transfer-Length"); @@ -215,5 +232,7 @@ bool LibFlute::RaptorFEC::add_fdt_info(tinyxml2::XMLElement *file) { file->SetAttribute("FEC-OTI-Number-Of-Sub-Blocks", N); file->SetAttribute("FEC-OTI-Symbol-Alignment-Parameter", Al); + is_encoder = true; + return true; } From 76da00690db9c2397d0ce4337be6ff64b6d454e3 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Tue, 17 Jan 2023 10:21:39 +0100 Subject: [PATCH 43/61] update raptor test.c --- raptor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raptor b/raptor index 0570760..9109793 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 0570760bf91d8050d5229e5fcb674c38f0b400a6 +Subproject commit 9109793b8471ab5162d67cd84052bc61022df2ed From 5a6b7ab4c6cb1fecee07d17a915b4ac04fa849cd Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Tue, 17 Jan 2023 10:23:39 +0100 Subject: [PATCH 44/61] fixed bad return in check_source_completion and allow fec transformres to allocate the file buffer themselves --- include/flute_types.h | 10 ++++++++++ src/File.cpp | 7 +++++-- src/RaptorFEC.cpp | 7 ++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 600b999..0753e12 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -127,6 +127,14 @@ namespace LibFlute { */ virtual bool add_fdt_info(tinyxml2::XMLElement *file) = 0; + /** + * @brief Allocate the size of the buffer needed for this encoding scheme (since it may be larger) + * + * @param min_length this should be the size of the file (transfer length). This determines the minimum size of the returned buffer + * @return 0 on failure, otherwise return a pointer to the buffer + */ + virtual void *allocate_file_buffer(int min_length) = 0; + uint32_t nof_source_symbols = 0; uint32_t nof_source_blocks = 0; uint32_t large_source_block_length = 0; @@ -171,6 +179,8 @@ namespace LibFlute { bool add_fdt_info(tinyxml2::XMLElement *file); + void *allocate_file_buffer(int min_length); + #ifdef RAPTOR_ENABLED std::map decoders; // map of source block number to decoders #endif diff --git a/src/File.cpp b/src/File.cpp index f2418aa..27a1699 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -34,7 +34,11 @@ LibFlute::File::File(LibFlute::FileDeliveryTable::FileEntry entry) spdlog::debug("Creating File from FileEntry"); // Allocate a data buffer spdlog::debug("Allocating buffer"); - _buffer = (char*)malloc(_meta.fec_oti.transfer_length); + if (_meta.fec_transformer){ + _buffer = (char*) _meta.fec_transformer->allocate_file_buffer(_meta.fec_oti.transfer_length); + } else { + _buffer = (char*) malloc(_meta.fec_oti.transfer_length); + } if (_buffer == nullptr) { throw "Failed to allocate file buffer"; @@ -147,7 +151,6 @@ auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> voi target_symbol.complete = true; if (_meta.fec_transformer) { _meta.fec_transformer->process_symbol(source_block,target_symbol,symbol.id()); - return; } check_source_block_completion(source_block); check_file_completion(); diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index bd39c1a..211e69d 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -61,6 +61,12 @@ void LibFlute::RaptorFEC::extract_finished_block(LibFlute::SourceBlock& srcblk, for(auto iter = srcblk.symbols.begin(); iter != srcblk.symbols.end(); iter++) { memcpy(iter->second.data,dc->pp[iter->first],T); // overwrite the encoded symbol with the source data; } + spdlog::debug("Raptor Decoder: finished decoding source block {}",srcblk.id); +} + +void *LibFlute::RaptorFEC::allocate_file_buffer(int min_length){ + assert(min_length <= Z*target_K()*T); // min length should be exactly Z*K*T, so including repair symbols we should be getting a larger value + return malloc(Z*target_K()*T); } bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute::Symbol& symbol, unsigned int id) { @@ -103,7 +109,6 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s spdlog::error("Couldnt find raptor decoder for source block {}",srcblk.id); return false; } - spdlog::debug("raptor is finished? : {}",dc->finished); if (dc->finished) { extract_finished_block(srcblk,dc); } From 9ddc9b30dd4d9a3f5a7ca31e05b15db344e48a28 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Tue, 17 Jan 2023 10:24:07 +0100 Subject: [PATCH 45/61] simulate 1pcent packet loss during reception, but never on the FDT --- include/Receiver.h | 6 ++++-- src/Receiver.cpp | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/include/Receiver.h b/include/Receiver.h index 72f50f2..96e41f6 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -23,6 +23,7 @@ #include "FileDeliveryTable.h" #include "flute_types.h" +#define SIMULATE_PKT_LOSS namespace LibFlute { /** * FLUTE receiver class. Construct an instance of this to receive files from a FLUTE/ALC session. @@ -104,7 +105,8 @@ namespace LibFlute { completion_callback_t _completion_cb = nullptr; bool _running = true; - - // FecScheme _fec_scheme; +#ifdef SIMULATE_PKT_LOSS + unsigned int packets_dropped = 0; +#endif }; }; diff --git a/src/Receiver.cpp b/src/Receiver.cpp index 0e8ef7a..10ca2fe 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -76,7 +76,14 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er _files.emplace(alc.toi(), std::make_shared(fe)); } } - +#ifdef SIMULATE_PKT_LOSS + // only simulate packet loss with objects that arent the FDT + bool process = true; + if((rand() % 100) < 2) { + spdlog::warn("Simulating 1%% packet loss, dropping symbols. Total Packets dropped = {}",++packets_dropped); + process = false; + } +#endif if (_files.find(alc.toi()) != _files.end() && !_files[alc.toi()]->complete()) { auto encoding_symbols = LibFlute::EncodingSymbol::from_payload( _data + alc.header_length(), @@ -85,8 +92,15 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er alc.content_encoding()); for (const auto& symbol : encoding_symbols) { +#ifdef SIMULATE_PKT_LOSS + spdlog::debug("{} TOI {} SBN {} ID {}", process ? "receiver" : "dropped",alc.toi(), symbol.source_block_number(), symbol.id() ); + if(process) { + _files[alc.toi()]->put_symbol(symbol); + } +#else spdlog::debug("received TOI {} SBN {} ID {}", alc.toi(), symbol.source_block_number(), symbol.id() ); - _files[alc.toi()]->put_symbol(symbol); + _files[alc.toi()]->put_symbol(symbol); +#endif //SIMULATE_PKT_LOSS } auto file = _files[alc.toi()].get(); From 9f2f5e5932b510bbdf70249d7183f2f3fbcd9438 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 19 Jan 2023 09:58:46 +0100 Subject: [PATCH 46/61] added github action to test the build process --- .github/workflows/build.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..e64b315 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,19 @@ +name: build +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + with: + submodules: recursive + - name: install dependencies + run: sudo apt install -y ninja-build cmake libssl-dev libboost-all-dev libspdlog-dev libtinyxml2-dev libnl-3-dev gcc pkg-config libconfig++-dev + - run: cmake --version + - name: build.sh + run: ${{ github.workspace }}/build.sh + - name: transmitter link check + run: ${{ github.workspace }}/build/examples/flute-transmitter --help + - name: receiver link check + run: ${{ github.workspace }}/build/examples/flute-receiver --help From 63d2e8dc09242361fd3aeabda9c5986f0c173756 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 19 Jan 2023 10:03:35 +0100 Subject: [PATCH 47/61] use matrix of build and os options + fix compilation when raptor is disabled or ssl doesnt define ex2 on ubu20 --- .github/workflows/.build.yml.swp | Bin 0 -> 12288 bytes .github/workflows/build.yml | 22 +++++++++++++++------- include/flute_types.h | 4 ++-- src/File.cpp | 2 +- src/FileDeliveryTable.cpp | 4 ++++ 5 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/.build.yml.swp diff --git a/.github/workflows/.build.yml.swp b/.github/workflows/.build.yml.swp new file mode 100644 index 0000000000000000000000000000000000000000..f2931ce7f3871d0bda09ca1d092e69fd6406eabc GIT binary patch literal 12288 zcmeI2zi$*r6vxMi4jT|DX&xB4fw^;R3OEUY9oqCz%mfS}4B5~WEIMan;rC{insHD%Y4lV55pl_d2oQl01cKk@mnPWk%$Ma>pFDd8zBqMf#7kr* z0z`la5CI}U1c(3;AOb|-|3tuNW9)le(2;skr|aw3(6ye@g$NJL=85)EMgJ1Y=K7k5ETZuf`es74-tOjG9ChdEXSdX%GP-Km>>Y5g-CYfCvx) zBJlYHpdNWJEjah!0aQn98^sY@mqkLt!-v%=+os%$(o9R&EDz)ITnedrG6&8(QfI6R z#KAz6ot;MbNxwYG?>xUilDDyT)r=e=f)bFi^w(Y^pE7RXg^~_*yG@mW++d@FrZF6DGwrUrVs=M>&3-eID zgL)E~Ns~w;lyv)~09=Sun*{9PvOxpF>J63*u#1PwTl?O@6<(3>$kR2;#&*_O zEG>X0Z28POg`cbs!X{N*=Vj)qpxEt>+j^JPy?)s+jdy0mm)IcP4lu1hT=CkNIlXH|^DjpAjyU(CvPT5QM0SZPt decoders; // map of source block number to decoders -#endif uint32_t nof_source_symbols = 0; uint32_t nof_source_blocks = 0; @@ -202,5 +201,6 @@ namespace LibFlute { unsigned int P; // maximum payload size: e.g. 1436 for ipv4 over 802.3 }; +#endif }; diff --git a/src/File.cpp b/src/File.cpp index 27a1699..167e598 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -302,7 +302,7 @@ auto LibFlute::File::mark_completed(const std::vector& symbols, const EVP_MD* md = EVP_md5(); unsigned int md_len; - EVP_DigestInit_ex2(context, md, NULL); + EVP_DigestInit_ex(context, md, NULL); EVP_DigestUpdate(context, input, length); EVP_DigestFinal_ex(context, result, &md_len); EVP_MD_CTX_free(context); diff --git a/src/FileDeliveryTable.cpp b/src/FileDeliveryTable.cpp index 37c064c..c1b995a 100644 --- a/src/FileDeliveryTable.cpp +++ b/src/FileDeliveryTable.cpp @@ -25,9 +25,11 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, FecOti fec_ , _global_fec_oti( fec_oti ) { switch (fec_oti.encoding_id){ +#ifdef RAPTOR_ENABLED case FecScheme::Raptor: _fdt_fec_transformer = new RaptorFEC(); break; +#endif default: _fdt_fec_transformer = 0; break; @@ -124,10 +126,12 @@ LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffe FecTransformer *fec_transformer = 0; switch (encoding_id){ +#ifdef RAPTOR_ENABLED case (int) FecScheme::Raptor: fec_transformer = new RaptorFEC(); // corresponding delete calls in Receiver.cpp and destuctor function spdlog::debug("Received FDT entry for a raptor encoded file"); break; +#endif default: break; } From 8d27e4ce7e3cdf63220713e3e206fb030133f3aa Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Thu, 19 Jan 2023 10:08:25 +0100 Subject: [PATCH 48/61] remove vim swap files --- .github/workflows/.build.yml.swp | Bin 12288 -> 0 bytes .gitignore | 1 + 2 files changed, 1 insertion(+) delete mode 100644 .github/workflows/.build.yml.swp diff --git a/.github/workflows/.build.yml.swp b/.github/workflows/.build.yml.swp deleted file mode 100644 index f2931ce7f3871d0bda09ca1d092e69fd6406eabc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2zi$*r6vxMi4jT|DX&xB4fw^;R3OEUY9oqCz%mfS}4B5~WEIMan;rC{insHD%Y4lV55pl_d2oQl01cKk@mnPWk%$Ma>pFDd8zBqMf#7kr* z0z`la5CI}U1c(3;AOb|-|3tuNW9)le(2;skr|aw3(6ye@g$NJL=85)EMgJ1Y=K7k5ETZuf`es74-tOjG9ChdEXSdX%GP-Km>>Y5g-CYfCvx) zBJlYHpdNWJEjah!0aQn98^sY@mqkLt!-v%=+os%$(o9R&EDz)ITnedrG6&8(QfI6R z#KAz6ot;MbNxwYG?>xUilDDyT)r=e=f)bFi^w(Y^pE7RXg^~_*yG@mW++d@FrZF6DGwrUrVs=M>&3-eID zgL)E~Ns~w;lyv)~09=Sun*{9PvOxpF>J63*u#1PwTl?O@6<(3>$kR2;#&*_O zEG>X0Z28POg`cbs!X{N*=Vj)qpxEt>+j^JPy?)s+jdy0mm)IcP4lu1hT=CkNIlXH|^DjpAjyU(CvPT5QM0SZPt Date: Thu, 19 Jan 2023 10:16:53 +0100 Subject: [PATCH 49/61] clean up build workflow --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50d5c78..33fadb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,14 +14,14 @@ jobs: with: submodules: recursive - name: install dependencies - run: sudo apt install -y ninja-build cmake libssl-dev libboost-all-dev libspdlog-dev libtinyxml2-dev libnl-3-dev gcc pkg-config libconfig++-dev + run: sudo apt install -y gcc ninja-build cmake libssl-dev libboost-all-dev libspdlog-dev libtinyxml2-dev libnl-3-dev pkg-config libconfig++-dev - name: pre-cmake - run: cmake --version && mkdir build && echo `nproc` processing units available + run: cmake --version && mkdir ${{ github.workspace }}/build && echo `nproc` processing units available - name: cmake - run: cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_RAPTOR=${{ matrix.raptor }} + run: cmake -S ${{ github.workspace }} -B ${{ github.workspace }}/build -GNinja -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_RAPTOR=${{ matrix.raptor }} - name: build - run: cd build && ninja -j`nproc` - - name: transmitter linking quickcheck + run: cd ${{ github.workspace }}/build && ninja -j`nproc` + - name: check for runtime linking errors in transmitter run: ${{ github.workspace }}/build/examples/flute-transmitter --help - - name: receiver linking quickcheck + - name: check for runtime linking errors in receiver run: ${{ github.workspace }}/build/examples/flute-receiver --help From 1dee66a626c0a611432b000164e1ea51c54f49ca Mon Sep 17 00:00:00 2001 From: SteezyE Date: Fri, 20 Jan 2023 16:11:38 +0100 Subject: [PATCH 50/61] fixed double free error --- src/FileDeliveryTable.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/FileDeliveryTable.cpp b/src/FileDeliveryTable.cpp index c1b995a..5887805 100644 --- a/src/FileDeliveryTable.cpp +++ b/src/FileDeliveryTable.cpp @@ -19,7 +19,6 @@ #include #include "spdlog/spdlog.h" - LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, FecOti fec_oti) : _instance_id( instance_id ) , _global_fec_oti( fec_oti ) @@ -41,15 +40,8 @@ LibFlute::FileDeliveryTable::~FileDeliveryTable() { delete _fdt_fec_transformer; _fdt_fec_transformer = 0; } - for(auto &f : _file_entries) { - if (f.fec_transformer){ - delete f.fec_transformer; - f.fec_transformer = 0; - } - } } - LibFlute::FileDeliveryTable::FileDeliveryTable(uint32_t instance_id, char* buffer, size_t len) : _instance_id( instance_id ) { From b7a2f0c961f8224a38d9e5c9afd60e560aeb1e54 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Fri, 20 Jan 2023 13:22:48 +0100 Subject: [PATCH 51/61] option stop receiving after n files + minor fixes --- .github/workflows/build.yml | 2 +- .gitignore | 1 + examples/flute-receiver.cpp | 14 +++++++++++--- include/Receiver.h | 4 ++-- src/Receiver.cpp | 27 ++++++++++++++++----------- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33fadb0..27b4ec0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: matrix: build-type: [Release] raptor: [ON, OFF] - os: [ubuntu-22.04, ubuntu-20.04] + os: [ubuntu-latest, ubuntu-22.04, ubuntu-20.04] runs-on: ${{ matrix.os }} steps: - name: check out repository code diff --git a/.gitignore b/.gitignore index 4ac8000..bdeb7aa 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ cmake-build-debug/ html/ .vscode/ *.swp +flute_download_* diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index f63fa96..be980b3 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -55,6 +55,7 @@ static struct argp_option options[] = { // NOLINT 0}, {"download-dir", 'd', "Download directory", 0 , "Directory in which to store downloaded files, defaults to the current directory otherwise", 0}, {"fec", 'f', "FEC Scheme", 0, "Choose a scheme for Forward Error Correction. Compact No Code = 0, Raptor = 1 (default is 0)", 0}, + {"num-files", 'n', "Stop Receiving after n files", 0, "Stop the reception after n files have been received (default is to never stop)", 0}, {nullptr, 0, nullptr, 0, nullptr, 0}}; /** @@ -69,6 +70,7 @@ struct ft_arguments { unsigned log_level = 2; /**< log level */ char *download_dir = nullptr; unsigned fec = 0; /**< log level */ + unsigned nfiles = 0; /**< log level */ char **files; }; @@ -104,6 +106,9 @@ static auto parse_opt(int key, char *arg, struct argp_state *state) -> error_t { case 'd': arguments->download_dir = arg; break; + case 'n': + arguments->nfiles = static_cast(strtoul(arg, nullptr, 10)); + break; default: return ARGP_ERR_UNKNOWN; } @@ -161,7 +166,6 @@ auto main(int argc, char **argv) -> int { arguments.mcast_target, (short)arguments.mcast_port, 16, - // LibFlute::FecScheme(arguments.fec), io); // Configure IPSEC, if enabled @@ -171,7 +175,7 @@ auto main(int argc, char **argv) -> int { } receiver.register_completion_callback( - [=](std::shared_ptr file) { //NOLINT + [&](std::shared_ptr file) { //NOLINT spdlog::info("{} (TOI {}) has been received", file->meta().content_location, file->meta().toi); char *buf = (char*) calloc(256,1); @@ -194,12 +198,16 @@ auto main(int argc, char **argv) -> int { spdlog::error("Error opening file {} to store received object",buf); } free(buf); + if (file->meta().toi == arguments.nfiles) { + spdlog::warn("{} file(s) received. Stopping reception",arguments.nfiles); + receiver.stop(); + } }); // Start the IO service io.run(); } catch (std::exception ex ) { - spdlog::error("Exiting on unhandled exception: %s", ex.what()); + spdlog::error("Exiting on unhandled exception: {}", ex.what()); } exit: diff --git a/include/Receiver.h b/include/Receiver.h index 96e41f6..acc1c63 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -23,7 +23,7 @@ #include "FileDeliveryTable.h" #include "flute_types.h" -#define SIMULATE_PKT_LOSS +#define SIMULATED_PKT_LOSS 2 // simulate 2% packet loss namespace LibFlute { /** * FLUTE receiver class. Construct an instance of this to receive files from a FLUTE/ALC session. @@ -105,7 +105,7 @@ namespace LibFlute { completion_callback_t _completion_cb = nullptr; bool _running = true; -#ifdef SIMULATE_PKT_LOSS +#ifdef SIMULATED_PKT_LOSS unsigned int packets_dropped = 0; #endif }; diff --git a/src/Receiver.cpp b/src/Receiver.cpp index 10ca2fe..21005cd 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -55,7 +55,12 @@ auto LibFlute::Receiver::enable_ipsec(uint32_t spi, const std::string& key) -> v auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& error, size_t bytes_recvd) -> void { - if (!_running) return; + if (!_running) { +#ifdef SIMULATED_PKT_LOSS + spdlog::warn("Stopping reception: total packets dropped {}", packets_dropped); +#endif // SIMULATED_PKT_LOSS + return; + } if (!error) { @@ -72,16 +77,16 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er if (alc.toi() == 0 && (!_fdt || _fdt->instance_id() != alc.fdt_instance_id())) { if (_files.find(alc.toi()) == _files.end()) { - FileDeliveryTable::FileEntry fe{0, "", static_cast(alc.fec_oti().transfer_length), "", "", 0, alc.fec_oti()}; + FileDeliveryTable::FileEntry fe{0, "", static_cast(alc.fec_oti().transfer_length), "", "", 0, alc.fec_oti(), 0}; _files.emplace(alc.toi(), std::make_shared(fe)); } } -#ifdef SIMULATE_PKT_LOSS +#ifdef SIMULATED_PKT_LOSS // only simulate packet loss with objects that arent the FDT - bool process = true; - if((rand() % 100) < 2) { - spdlog::warn("Simulating 1%% packet loss, dropping symbols. Total Packets dropped = {}",++packets_dropped); - process = false; + bool process_pkt = true; + if((rand() % 100) < SIMULATED_PKT_LOSS) { + spdlog::warn("Simulating {}% packet loss, dropping symbols. Total Packets dropped = {}",SIMULATED_PKT_LOSS,++packets_dropped); + process_pkt = false; } #endif if (_files.find(alc.toi()) != _files.end() && !_files[alc.toi()]->complete()) { @@ -92,15 +97,15 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er alc.content_encoding()); for (const auto& symbol : encoding_symbols) { -#ifdef SIMULATE_PKT_LOSS - spdlog::debug("{} TOI {} SBN {} ID {}", process ? "receiver" : "dropped",alc.toi(), symbol.source_block_number(), symbol.id() ); - if(process) { +#ifdef SIMULATED_PKT_LOSS + spdlog::debug("{} TOI {} SBN {} ID {}", process_pkt ? "receiver" : "dropped",alc.toi(), symbol.source_block_number(), symbol.id() ); + if(process_pkt) { _files[alc.toi()]->put_symbol(symbol); } #else spdlog::debug("received TOI {} SBN {} ID {}", alc.toi(), symbol.source_block_number(), symbol.id() ); _files[alc.toi()]->put_symbol(symbol); -#endif //SIMULATE_PKT_LOSS +#endif //SIMULATED_PKT_LOSS } auto file = _files[alc.toi()].get(); From 5abb85e3c0c960a6ef8bf31c2374ab8d63253d9b Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Sat, 21 Jan 2023 18:20:35 +0100 Subject: [PATCH 52/61] use mmap to read in files, dont simulate packet loss if there is no active FEC scheme --- examples/flute-receiver.cpp | 9 --------- examples/flute-transmitter.cpp | 36 +++++++++++++++++++++++----------- include/Receiver.h | 3 +++ include/flute_types.h | 2 +- raptor | 2 +- src/RaptorFEC.cpp | 5 ++++- src/Receiver.cpp | 6 +++--- 7 files changed, 37 insertions(+), 26 deletions(-) diff --git a/examples/flute-receiver.cpp b/examples/flute-receiver.cpp index be980b3..a3df945 100644 --- a/examples/flute-receiver.cpp +++ b/examples/flute-receiver.cpp @@ -54,7 +54,6 @@ static struct argp_option options[] = { // NOLINT "critical, 6 = none. Default: 2.", 0}, {"download-dir", 'd', "Download directory", 0 , "Directory in which to store downloaded files, defaults to the current directory otherwise", 0}, - {"fec", 'f', "FEC Scheme", 0, "Choose a scheme for Forward Error Correction. Compact No Code = 0, Raptor = 1 (default is 0)", 0}, {"num-files", 'n', "Stop Receiving after n files", 0, "Stop the reception after n files have been received (default is to never stop)", 0}, {nullptr, 0, nullptr, 0, nullptr, 0}}; @@ -69,7 +68,6 @@ struct ft_arguments { unsigned short mcast_port = 40085; unsigned log_level = 2; /**< log level */ char *download_dir = nullptr; - unsigned fec = 0; /**< log level */ unsigned nfiles = 0; /**< log level */ char **files; }; @@ -96,13 +94,6 @@ static auto parse_opt(int key, char *arg, struct argp_state *state) -> error_t { case 'l': arguments->log_level = static_cast(strtoul(arg, nullptr, 10)); break; - case 'f': - arguments->fec = static_cast(strtoul(arg, nullptr, 10)); - if ( (arguments->fec | 1) != 1 ) { - spdlog::error("Invalid FEC scheme! Please pick either 0 (Compact No Code) or 1 (Raptor)"); - return ARGP_ERR_UNKNOWN; - } - break; case 'd': arguments->download_dir = arg; break; diff --git a/examples/flute-transmitter.cpp b/examples/flute-transmitter.cpp index d544802..7ede617 100644 --- a/examples/flute-transmitter.cpp +++ b/examples/flute-transmitter.cpp @@ -16,9 +16,9 @@ #include #include #include - #include - +#include +#include #include #include #include @@ -166,15 +166,29 @@ auto main(int argc, char **argv) -> int { // read the file contents into the buffers for (int j = 0; arguments.files[j]; j++) { - std::string location = arguments.files[j]; - std::ifstream file(arguments.files[j], std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - if (size > 0) { - char* buffer = (char*)malloc(size); - file.read(buffer, size); - files.push_back(FsFile{ arguments.files[j], buffer, (size_t)size}); + struct stat sb; + int fd; + fd = open(arguments.files[j], O_RDONLY); + if (fd == -1) { + spdlog::error("Couldnt open file {}",arguments.files[j]); + continue; + } + if (fstat(fd, &sb) == -1){ // To obtain file size + spdlog::error("fstat() call for file {} failed",arguments.files[j]); + close(fd); + continue; + } + if (sb.st_size <= 0) { + close(fd); + continue; + } + char* buffer = (char*) mmap(nullptr, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if ( (long) buffer <= 0) { + spdlog::error("mmap() failed for file {}",arguments.files[j]); + continue; } + files.push_back(FsFile{ arguments.files[j], buffer, (size_t) sb.st_size}); } // Create a Boost io_service @@ -202,7 +216,7 @@ auto main(int argc, char **argv) -> int { for (auto& file : files) { if (file.toi == toi) { spdlog::info("{} (TOI {}) has been transmitted", file.location,file.toi); - // could free() the buffer here + munmap(file.buffer,file.len); } } }); diff --git a/include/Receiver.h b/include/Receiver.h index acc1c63..6771084 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -23,7 +23,10 @@ #include "FileDeliveryTable.h" #include "flute_types.h" +#ifdef RAPTOR_ENABLED #define SIMULATED_PKT_LOSS 2 // simulate 2% packet loss +#endif + namespace LibFlute { /** * FLUTE receiver class. Construct an instance of this to receive files from a FLUTE/ALC session. diff --git a/include/flute_types.h b/include/flute_types.h index 6a74112..1ade49f 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -156,7 +156,7 @@ namespace LibFlute { LibFlute::SourceBlock create_block(char *buffer, int *bytes_read, int blockid); - const float surplus_packet_ratio = 1.5; + const float surplus_packet_ratio = 1.15; // adds 15% transmission overhead in exchange for protection against up to 15% packet loss. Assuming 1 symbol per packet, for smaller files packets may contain up to 10 symbols per packet but small files are much less vulnerable to packet loss anyways void extract_finished_block(LibFlute::SourceBlock& srcblk, struct dec_context *dc); diff --git a/raptor b/raptor index 9109793..7f8fddc 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 9109793b8471ab5162d67cd84052bc61022df2ed +Subproject commit 7f8fddcbadd3cda1dd3bf99fc376104551fd91eb diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 211e69d..90f78d4 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -115,7 +115,10 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s return dc->finished; } -unsigned int LibFlute::RaptorFEC::target_K() { return K * surplus_packet_ratio; } +unsigned int LibFlute::RaptorFEC::target_K() { + int target = K * surplus_packet_ratio; + return target > K ? target : K + 1; // always send at least one repair symbol +} LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx){ // TODO: Delete in the File destructor (or anywhere where applicable) diff --git a/src/Receiver.cpp b/src/Receiver.cpp index 21005cd..911c527 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -82,9 +82,9 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er } } #ifdef SIMULATED_PKT_LOSS - // only simulate packet loss with objects that arent the FDT + // only simulate packet loss with objects that arent the FDT and if we have an FEC scheme active) bool process_pkt = true; - if((rand() % 100) < SIMULATED_PKT_LOSS) { + if(_files[alc.toi()]->meta().fec_transformer && rand() % 100 < SIMULATED_PKT_LOSS) { spdlog::warn("Simulating {}% packet loss, dropping symbols. Total Packets dropped = {}",SIMULATED_PKT_LOSS,++packets_dropped); process_pkt = false; } @@ -98,7 +98,7 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er for (const auto& symbol : encoding_symbols) { #ifdef SIMULATED_PKT_LOSS - spdlog::debug("{} TOI {} SBN {} ID {}", process_pkt ? "receiver" : "dropped",alc.toi(), symbol.source_block_number(), symbol.id() ); + spdlog::debug("{} TOI {} SBN {} ID {}", process_pkt ? "received" : "dropped",alc.toi(), symbol.source_block_number(), symbol.id() ); if(process_pkt) { _files[alc.toi()]->put_symbol(symbol); } From 99082df0a642af3f75368d9358914e124e2b22a6 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Tue, 24 Jan 2023 19:55:40 +0100 Subject: [PATCH 53/61] Fix error with accessing FDT under simulated packet loss --- src/Receiver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Receiver.cpp b/src/Receiver.cpp index 911c527..7aad5d7 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -84,7 +84,7 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er #ifdef SIMULATED_PKT_LOSS // only simulate packet loss with objects that arent the FDT and if we have an FEC scheme active) bool process_pkt = true; - if(_files[alc.toi()]->meta().fec_transformer && rand() % 100 < SIMULATED_PKT_LOSS) { + if(alc.toi() && _files[alc.toi()]->meta().fec_transformer && rand() % 100 < SIMULATED_PKT_LOSS) { spdlog::warn("Simulating {}% packet loss, dropping symbols. Total Packets dropped = {}",SIMULATED_PKT_LOSS,++packets_dropped); process_pkt = false; } From 1d9df0a3137e0d4f68d7202f467abd955bfd2947 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Tue, 24 Jan 2023 19:56:19 +0100 Subject: [PATCH 54/61] Free the Symbol data --- src/RaptorFEC.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 90f78d4..4dc49d1 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -95,7 +95,12 @@ bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { if (is_encoder) { // check source block completion for the Encoder - return std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); + bool complete = std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); + + if(complete) + std::for_each(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ delete symbol.second.data; }); + + return complete; } // else case- we are the Decoder @@ -120,8 +125,7 @@ unsigned int LibFlute::RaptorFEC::target_K() { return target > K ? target : K + 1; // always send at least one repair symbol } -LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx){ - // TODO: Delete in the File destructor (or anywhere where applicable) +LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx){ struct LT_packet *lt_packet = encode_LT_packet(encoder_ctx); struct Symbol symbol { new char[T], T}; From 375f31ab4d9b1623832ceecf524cd8b628cdf620 Mon Sep 17 00:00:00 2001 From: Dominik Danelski Date: Fri, 10 Feb 2023 17:44:09 +0100 Subject: [PATCH 55/61] Get rid of code-simulated paket loss --- include/Receiver.h | 4 ---- src/Receiver.cpp | 17 ++--------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/include/Receiver.h b/include/Receiver.h index 6771084..b01f52e 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -23,10 +23,6 @@ #include "FileDeliveryTable.h" #include "flute_types.h" -#ifdef RAPTOR_ENABLED -#define SIMULATED_PKT_LOSS 2 // simulate 2% packet loss -#endif - namespace LibFlute { /** * FLUTE receiver class. Construct an instance of this to receive files from a FLUTE/ALC session. diff --git a/src/Receiver.cpp b/src/Receiver.cpp index 7aad5d7..4152dfa 100644 --- a/src/Receiver.cpp +++ b/src/Receiver.cpp @@ -81,14 +81,7 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er _files.emplace(alc.toi(), std::make_shared(fe)); } } -#ifdef SIMULATED_PKT_LOSS - // only simulate packet loss with objects that arent the FDT and if we have an FEC scheme active) - bool process_pkt = true; - if(alc.toi() && _files[alc.toi()]->meta().fec_transformer && rand() % 100 < SIMULATED_PKT_LOSS) { - spdlog::warn("Simulating {}% packet loss, dropping symbols. Total Packets dropped = {}",SIMULATED_PKT_LOSS,++packets_dropped); - process_pkt = false; - } -#endif + if (_files.find(alc.toi()) != _files.end() && !_files[alc.toi()]->complete()) { auto encoding_symbols = LibFlute::EncodingSymbol::from_payload( _data + alc.header_length(), @@ -97,15 +90,9 @@ auto LibFlute::Receiver::handle_receive_from(const boost::system::error_code& er alc.content_encoding()); for (const auto& symbol : encoding_symbols) { -#ifdef SIMULATED_PKT_LOSS - spdlog::debug("{} TOI {} SBN {} ID {}", process_pkt ? "received" : "dropped",alc.toi(), symbol.source_block_number(), symbol.id() ); - if(process_pkt) { - _files[alc.toi()]->put_symbol(symbol); - } -#else + spdlog::debug("received TOI {} SBN {} ID {}", alc.toi(), symbol.source_block_number(), symbol.id() ); _files[alc.toi()]->put_symbol(symbol); -#endif //SIMULATED_PKT_LOSS } auto file = _files[alc.toi()].get(); From efccec1d08aa83e21af142d912c8a84486ea8260 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Fri, 10 Feb 2023 19:07:29 +0100 Subject: [PATCH 56/61] scripts for adding packet loss to loopback interface for testing --- reset_loopback_settings | 14 ++++++++++++++ setup_packet_loss_on_loopback | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 reset_loopback_settings create mode 100755 setup_packet_loss_on_loopback diff --git a/reset_loopback_settings b/reset_loopback_settings new file mode 100755 index 0000000..5a8ebfc --- /dev/null +++ b/reset_loopback_settings @@ -0,0 +1,14 @@ +#! /bin/bash + +t=238.1.1.95 + +echo Removing route to target address +sudo ip r del $t dev lo +echo Removing packet loss from loopback device +sudo tc qdisc del dev lo root netem loss 2% + +echo Check everything got removed +ip r get $t +tc qdisc show dev lo +echo 2 second ping check to assert packet loss is gone +ping -I lo 127.0.0.1 -i 0.002 -w 2 -q diff --git a/setup_packet_loss_on_loopback b/setup_packet_loss_on_loopback new file mode 100755 index 0000000..708117c --- /dev/null +++ b/setup_packet_loss_on_loopback @@ -0,0 +1,18 @@ +#! /bin/bash + +t=238.1.1.95 + +if [[ -z $(ip r get $t | grep 'dev lo') ]]; then + echo Routing default target address via loopback device + sudo ip r add $t dev lo +else + echo Looks like this is already routed via loopback +fi +echo Adding packet loss to loopback device +sudo tc qdisc add dev lo root netem loss 2% + +echo Check everything worked +ip r get $t +tc qdisc show dev lo +echo 2 second ping check to assert packet loss is working +ping -I lo 127.0.0.1 -i 0.002 -w 2 -q From 4c07c035375f2568f9907ebcf73b010644d40c82 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Fri, 10 Feb 2023 22:16:00 +0100 Subject: [PATCH 57/61] deal with the last block case --- include/flute_types.h | 3 +- raptor | 2 +- src/File.cpp | 5 +-- src/RaptorFEC.cpp | 75 ++++++++++++++++++++++++++++--------------- 4 files changed, 56 insertions(+), 29 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 1ade49f..905230b 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -150,7 +150,7 @@ namespace LibFlute { bool is_encoder = true; - unsigned int target_K(); + unsigned int target_K(int blockno); Symbol translate_symbol(struct enc_context *encoder_ctx); @@ -198,6 +198,7 @@ namespace LibFlute { unsigned int Z; // number of source blocks unsigned int N; // number of sub-blocks per source block unsigned int K; // number of symbols in a source block + unsigned int Kt; // total number of symbols unsigned int P; // maximum payload size: e.g. 1436 for ipv4 over 802.3 }; diff --git a/raptor b/raptor index 7f8fddc..015735f 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 7f8fddcbadd3cda1dd3bf99fc376104551fd91eb +Subproject commit 015735fc86d283c2bac0634912c88c3a0b0f4a04 diff --git a/src/File.cpp b/src/File.cpp index 167e598..78b95a1 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -236,7 +236,7 @@ auto LibFlute::File::create_blocks() -> void auto symbol_length = std::min(remaining_size, (size_t)_meta.fec_oti.encoding_symbol_length); assert(buffer_ptr + symbol_length <= _buffer + _meta.fec_oti.transfer_length); - LibFlute::Symbol symbol{.data = buffer_ptr, .length = symbol_length, .complete = false}; + LibFlute::Symbol symbol{ .data = buffer_ptr, .length = symbol_length, .complete = false}; block.symbols[ symbol_id++ ] = symbol; remaining_size -= symbol_length; @@ -294,9 +294,10 @@ auto LibFlute::File::mark_completed(const std::vector& symbols, { // simple implementation based on openssl docs (https://www.openssl.org/docs/man3.0/man3/EVP_DigestInit_ex.html) if (!input || ! length) { - spdlog::debug("MD5 called for empty input (input {}, length {})", input, length); + spdlog::error("MD5 called with invalid input"); return -1; } + spdlog::debug("MD5 calculation called for input length {}", length); EVP_MD_CTX* context = EVP_MD_CTX_new(); const EVP_MD* md = EVP_md5(); diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 4dc49d1..c974781 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -13,22 +13,31 @@ LibFlute::RaptorFEC::RaptorFEC(unsigned int transfer_length, unsigned int max_pa spdlog::debug("T = (unsigned int) floor((double)P/(double)(Al*g)) * Al"); spdlog::debug("T = {} = floor({}/({}*{})) * {}",T,P,Al,g,Al); - assert(T % Al == 0); // Symbol size T should be a multiple of symbol alignment parameter Al + if (T % Al){ + spdlog::error(" Symbol size T should be a multiple of symbol alignment parameter Al"); + throw "Symbol size doesnt align"; + } - double Kt = ceil((double)F/(double)T); // total symbols + Kt = ceil((double)F/(double)T); // total symbols spdlog::debug("double Kt = ceil((double)F/(double)T)"); spdlog::debug("Kt = {} = ceil({}/{})",Kt,F,T); - Z = (unsigned int) ceil(Kt/8192); + if (Kt < 4){ + spdlog::error("Input file is too small, it must be a minimum of 4 Symbols"); + throw "Input is less than 4 symbols"; + } + + Z = (unsigned int) ceil((double)Kt/(double)8192); spdlog::debug("Z = (unsigned int) ceil(Kt/8192)"); spdlog::debug("Z = {} = ceil({}/8192)",Z,Kt); K = (Kt > 8192) ? 8192 : (unsigned int) Kt; // symbols per source block spdlog::debug("K = {}",K); - N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al ); - spdlog::warn("N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al )"); - spdlog::warn("N = {} = min( ceil( ceil({}/{}) * {}/{} ) , {}/{} )",N,Kt,Z,T,W,T,Al); + N = fmin( ceil( ceil((double)Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al ); + spdlog::debug("N = fmin( ceil( ceil(Kt/(double)Z) * (double)T/(double)W ) , (double)T/(double)Al )"); + spdlog::debug("N = {} = min( ceil( ceil({}/{}) * {}/{} ) , {}/{} )",N,Kt,Z,T,W,T,Al); + // Set the values that the File class may need: nof_source_symbols = (unsigned int) Kt; @@ -65,15 +74,19 @@ void LibFlute::RaptorFEC::extract_finished_block(LibFlute::SourceBlock& srcblk, } void *LibFlute::RaptorFEC::allocate_file_buffer(int min_length){ - assert(min_length <= Z*target_K()*T); // min length should be exactly Z*K*T, so including repair symbols we should be getting a larger value - return malloc(Z*target_K()*T); + assert(min_length <= Z*target_K(0)*T); // min length should be exactly Z*K*T, so including repair symbols we should have a larger value + return malloc(Z*target_K(0)*T); } bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute::Symbol& symbol, unsigned int id) { - assert(symbol.length == T); // symbol.length should always be T (the symbol's size) + assert(symbol.length == T); // symbol.length should always be the symbol size, T struct dec_context *dc = decoders[srcblk.id]; if (!dc) { - struct enc_context *sc = create_encoder_context(NULL, K, T, srcblk.id); + int nsymbs = (srcblk.id < Z - 1) ? K : Kt - K*(Z-1); // the last block will usually be smaller than the normal block size, unless the file size is an exact multiple + int blocksize = (srcblk.id < Z - 1) ? K*T : F - K*T*(Z-1); + spdlog::debug("Preparing decoder context with {} blocks and blocksize {}", nsymbs, blocksize); + struct enc_context *sc = create_encoder_context(NULL, nsymbs, T, blocksize, srcblk.id); + //struct enc_context *sc = create_encoder_context(NULL, K, T, K*T, srcblk.id); // the "length" will always be K*T from the decoders perspective dc = create_decoder_context(sc); decoders[srcblk.id] = dc; } @@ -83,11 +96,14 @@ bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute } struct LT_packet * pkt = (struct LT_packet *) calloc(1, sizeof(struct LT_packet)); pkt->id = id; - pkt->syms = (GF_ELEMENT *) calloc(symbol.length, sizeof(char)); + pkt->syms = (GF_ELEMENT *) malloc(symbol.length * sizeof(char)); memcpy(pkt->syms, symbol.data, symbol.length * sizeof(char)); process_LT_packet(dc, pkt); free_LT_packet(pkt); + if (dc->finished) { + extract_finished_block(srcblk, dc); + } return true; } @@ -114,15 +130,18 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s spdlog::error("Couldnt find raptor decoder for source block {}",srcblk.id); return false; } - if (dc->finished) { - extract_finished_block(srcblk,dc); - } return dc->finished; } -unsigned int LibFlute::RaptorFEC::target_K() { - int target = K * surplus_packet_ratio; - return target > K ? target : K + 1; // always send at least one repair symbol +unsigned int LibFlute::RaptorFEC::target_K(int blockno) { + // always send at least one repair symbol + if (blockno < Z-1) { + int target = K * surplus_packet_ratio; + return (target > K) ? target : K + 1; + } + // last block gets special treatment + int remaining_symbs = Kt - K*(Z-1); + return (remaining_symbs + 1 > remaining_symbs*surplus_packet_ratio) ? remaining_symbs + 1 : remaining_symbs * surplus_packet_ratio; } LibFlute::Symbol LibFlute::RaptorFEC::translate_symbol(struct enc_context *encoder_ctx){ @@ -139,13 +158,20 @@ LibFlute::SourceBlock LibFlute::RaptorFEC::create_block(char *buffer, int *bytes struct SourceBlock source_block; source_block.id = blockid; int seed = blockid; - struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T, seed); - unsigned int symbols_to_read = target_K(); - + int nsymbs = (blockid < Z - 1) ? K : Kt - K*(Z-1); + int blocksize = (blockid < Z - 1) ? K*T : F - K*T*(Z-1); // the last block will usually be smaller than the normal block size, unless the file size is an exact multiple + struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, nsymbs , T, blocksize, seed); +// struct enc_context *encoder_ctx = create_encoder_context((unsigned char *)buffer, K, T, blocksize, seed); + if (!encoder_ctx) { + spdlog::error("Error creating encoder context"); + throw "Error creating encoder context"; + } + unsigned int symbols_to_read = target_K(blockid); for(unsigned int symbol_id = 0; symbol_id < symbols_to_read; symbol_id++) { source_block.symbols[symbol_id] = translate_symbol(encoder_ctx); } - *bytes_read += K * T; + *bytes_read += blocksize; + spdlog::debug("Creating encoder context with {} blocks and blocksize {}", nsymbs, blocksize); free_encoder_context(encoder_ctx); return source_block; @@ -164,18 +190,16 @@ std::map LibFlute::RaptorFEC::create_blocks(cha for(unsigned int src_blocks = 0; src_blocks < Z; src_blocks++) { if(!is_encoder) { LibFlute::SourceBlock block; - unsigned int symbols_to_read = target_K(); + unsigned int symbols_to_read = target_K(src_blocks); for (int i = 0; i < symbols_to_read; i++) { block.symbols[i] = Symbol {.data = buffer + T*i, .length = T, .complete = false}; } + block.id = src_blocks; block_map[src_blocks] = block; } else { block_map[src_blocks] = create_block(&buffer[*bytes_read], bytes_read, src_blocks); } } - if(!is_encoder) { - spdlog::debug("Raptor Decoder- prepared {} empty source blocks with {} symbols (K = {})",block_map.size(),target_K(),K); - } return block_map; } @@ -226,6 +250,7 @@ bool LibFlute::RaptorFEC::parse_fdt_info(tinyxml2::XMLElement *file) { // Set the values that are missing that we or the File class may need, follows the same logic as in calculate_partitioning() nof_source_symbols = ceil((double)F / (double)T); K = (nof_source_symbols > 8192) ? 8192 : nof_source_symbols; + Kt = ceil((double)F/(double)T); // total symbols nof_source_blocks = Z; small_source_block_length = (Z * K - nof_source_symbols) * T; From 2db05611b4814ace8c693a8c542139783994615a Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Sat, 11 Feb 2023 17:11:43 +0100 Subject: [PATCH 58/61] calculate buffer offset correctly for multi block files + use delete[] instead of delete --- raptor | 2 +- src/File.cpp | 11 ++++++++--- src/RaptorFEC.cpp | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/raptor b/raptor index 015735f..2544f6e 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 015735fc86d283c2bac0634912c88c3a0b0f4a04 +Subproject commit 2544f6e712b98846cc730c00f511223007fb270a diff --git a/src/File.cpp b/src/File.cpp index 78b95a1..cd65392 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -131,7 +131,7 @@ LibFlute::File::~File() auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> void { if(_complete) { - spdlog::debug("Skipped processing symbol {} , SBN {} since file is already complete",symbol.id(),symbol.source_block_number()); + spdlog::debug("Not handling symbol {} , SBN {} since file is already complete",symbol.id(),symbol.source_block_number()); return; } if (symbol.source_block_number() > _source_blocks.size()) { @@ -140,6 +140,11 @@ auto LibFlute::File::put_symbol( const LibFlute::EncodingSymbol& symbol ) -> voi SourceBlock& source_block = _source_blocks[ symbol.source_block_number() ]; + if(source_block.complete){ + spdlog::warn("Ignoring symbol {} since block {} is already complete",symbol.id(),symbol.source_block_number()); + return; + } + if (symbol.id() > source_block.symbols.size()) { throw "Encoding Symbol ID too high"; } @@ -178,7 +183,7 @@ auto LibFlute::File::check_file_completion() -> void auto content_md5 = base64_decode(_meta.content_md5); if (memcmp(md5, content_md5.c_str(), MD5_DIGEST_LENGTH) != 0) { - spdlog::warn("MD5 mismatch for TOI {}, discarding", _meta.toi); + spdlog::error("MD5 mismatch for TOI {}, discarding", _meta.toi); // MD5 mismatch, try again for (auto& block : _source_blocks) { @@ -186,7 +191,7 @@ auto LibFlute::File::check_file_completion() -> void symbol.second.complete = false; } block.second.complete = false; - } + } _complete = false; } } diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index c974781..5744252 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -114,7 +114,7 @@ bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& s bool complete = std::all_of(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ return symbol.second.complete; }); if(complete) - std::for_each(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ delete symbol.second.data; }); + std::for_each(srcblk.symbols.begin(), srcblk.symbols.end(), [](const auto& symbol){ delete[] symbol.second.data; }); return complete; } @@ -192,7 +192,7 @@ std::map LibFlute::RaptorFEC::create_blocks(cha LibFlute::SourceBlock block; unsigned int symbols_to_read = target_K(src_blocks); for (int i = 0; i < symbols_to_read; i++) { - block.symbols[i] = Symbol {.data = buffer + T*i, .length = T, .complete = false}; + block.symbols[i] = Symbol {.data = buffer + src_blocks*K*T + T*i, .length = T, .complete = false}; } block.id = src_blocks; block_map[src_blocks] = block; From 730d3a2382924c5b59c2786e7516003ba235adb0 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Sun, 12 Feb 2023 13:55:39 +0100 Subject: [PATCH 59/61] only extract finished source blocks at the end of transmission --- include/flute_types.h | 9 +++++++++ src/File.cpp | 5 +++++ src/RaptorFEC.cpp | 18 ++++++++++++------ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/include/flute_types.h b/include/flute_types.h index 905230b..fa8139e 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -135,6 +135,13 @@ namespace LibFlute { */ virtual void *allocate_file_buffer(int min_length) = 0; + /** + * @brief Called after the file is marked as complete, to finish extraction/decoding (if necessary) + * + * @param blocks the source blocks of the file, stored in the File object + */ + virtual bool extract_file(std::map blocks) = 0; + uint32_t nof_source_symbols = 0; uint32_t nof_source_blocks = 0; uint32_t large_source_block_length = 0; @@ -182,6 +189,8 @@ namespace LibFlute { void *allocate_file_buffer(int min_length); + bool extract_file(std::map blocks); + std::map decoders; // map of source block number to decoders uint32_t nof_source_symbols = 0; diff --git a/src/File.cpp b/src/File.cpp index cd65392..ab0fb23 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -177,6 +177,11 @@ auto LibFlute::File::check_file_completion() -> void _complete = std::all_of(_source_blocks.begin(), _source_blocks.end(), [](const auto& block){ return block.second.complete; }); if (_complete && !_meta.content_md5.empty()) { + + if(_meta.fec_transformer){ + _meta.fec_transformer->extract_file(_source_blocks); + } + //check MD5 sum unsigned char md5[EVP_MAX_MD_SIZE]; calculate_md5(buffer(),length(),md5); diff --git a/src/RaptorFEC.cpp b/src/RaptorFEC.cpp index 5744252..ea3b8be 100644 --- a/src/RaptorFEC.cpp +++ b/src/RaptorFEC.cpp @@ -67,9 +67,12 @@ bool LibFlute::RaptorFEC::calculate_partitioning() { } void LibFlute::RaptorFEC::extract_finished_block(LibFlute::SourceBlock& srcblk, struct dec_context *dc) { - for(auto iter = srcblk.symbols.begin(); iter != srcblk.symbols.end(); iter++) { - memcpy(iter->second.data,dc->pp[iter->first],T); // overwrite the encoded symbol with the source data; - } + if(!dc){ + return; + } + for(auto iter = srcblk.symbols.begin(); iter != srcblk.symbols.end(); iter++) { + memcpy(iter->second.data,dc->pp[iter->first],T); // overwrite the encoded symbol with the source data; + } spdlog::debug("Raptor Decoder: finished decoding source block {}",srcblk.id); } @@ -101,12 +104,15 @@ bool LibFlute::RaptorFEC::process_symbol(LibFlute::SourceBlock& srcblk, LibFlute process_LT_packet(dc, pkt); free_LT_packet(pkt); - if (dc->finished) { - extract_finished_block(srcblk, dc); - } return true; } +bool LibFlute::RaptorFEC::extract_file(std::map blocks) { + for(auto iter = blocks.begin(); iter != blocks.end(); iter++) { + extract_finished_block(iter->second,decoders[iter->second.id]); + } + return true; +} bool LibFlute::RaptorFEC::check_source_block_completion(LibFlute::SourceBlock& srcblk) { if (is_encoder) { From 28c034e71b9cd1bab0506dcfbfced1531e7aaa82 Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Wed, 8 Mar 2023 13:53:11 +0100 Subject: [PATCH 60/61] small fixes: memory deallocation in raptor and derived class destructor calls in FEC + updated readme --- README.md | 20 +++++++++++++++++++- include/flute_types.h | 6 +++++- raptor | 2 +- src/EncodingSymbol.cpp | 2 ++ src/File.cpp | 2 +- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1e1cafc..eca806f 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,16 @@ git submodule update --init ### Step 2: Building ```` cd rt-libflute/ -./build.sh +mkdir build +cd build +cmake .. +cmake --build . ```` +(alternatively you can use the `build.sh` script to build in debug mode with raptor enabled) + +Build options: + +- "Enable Raptor": build with support for the raptor10-based forward error correction. On by default. Disable by passing the `-DENABLE_RAPTOR=OFF` option to cmake ## Usage @@ -88,6 +96,16 @@ sudo setcap 'cap_net_admin=eip' ./flute-transmitter sudo setcap 'cap_net_admin=eip' ./flute-receiver ```` +### Optional: Forward Error Correction (FEC) for lossy environment + +To use forward error correction to overcome packet loss during transmission add the option `-f 1` to the transmitter. The receiver needs no such option, just make sure that both of them were properly built/rebuilt with raptor enabled in the build options (this is the default). + +To simulate packet loss over the loopback interface and test that FEC works, you can run the `setup_packet_loss_on_loopback` script, as root. + +Then start a transmission as you usually would. Make sure to use the default ip address, since the script will reroute this to go through the loopback interface. + +Finally revert the changes to your loopback interface and routing table with the `reset_loopback_settings` script. + ## Documentation Documentation of the source code can be found at: https://5g-mag.github.io/rt-libflute/ diff --git a/include/flute_types.h b/include/flute_types.h index fa8139e..da70ce0 100644 --- a/include/flute_types.h +++ b/include/flute_types.h @@ -82,8 +82,12 @@ namespace LibFlute { * abstract class for FEC Object En/De-coding */ class FecTransformer { + public: + virtual ~FecTransformer() = default; + + /** * @brief Attempt to decode a source block * @@ -173,7 +177,7 @@ namespace LibFlute { RaptorFEC() {}; - virtual ~RaptorFEC(); + ~RaptorFEC(); bool check_source_block_completion(SourceBlock& srcblk); diff --git a/raptor b/raptor index 2544f6e..4ee8bf2 160000 --- a/raptor +++ b/raptor @@ -1 +1 @@ -Subproject commit 2544f6e712b98846cc730c00f511223007fb270a +Subproject commit 4ee8bf2f154670007bde69c8cbbd2db57f938aa3 diff --git a/src/EncodingSymbol.cpp b/src/EncodingSymbol.cpp index a0abd1e..553b5aa 100644 --- a/src/EncodingSymbol.cpp +++ b/src/EncodingSymbol.cpp @@ -48,6 +48,7 @@ auto LibFlute::EncodingSymbol::from_payload(char* encoded_data, size_t data_len, int nof_symbols = std::ceil((float)data_len / (float)fec_oti.encoding_symbol_length); for (int i = 0; i < nof_symbols; i++) { switch(fec_oti.encoding_id) { + default: case FecScheme::CompactNoCode: case FecScheme::Raptor: symbols.emplace_back(encoding_symbol_id, source_block_number, encoded_data, std::min(data_len, (size_t)fec_oti.encoding_symbol_length), fec_oti.encoding_id); @@ -107,6 +108,7 @@ auto LibFlute::EncodingSymbol::decode_to(char* buffer, size_t max_length) const auto LibFlute::EncodingSymbol::encode_to(char* buffer, size_t max_length) const -> size_t { switch (_fec_scheme) { + default: case FecScheme::CompactNoCode: case FecScheme::Raptor: if (_data_len <= max_length) { diff --git a/src/File.cpp b/src/File.cpp index ab0fb23..661dd32 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -318,7 +318,7 @@ auto LibFlute::File::mark_completed(const std::vector& symbols, EVP_DigestFinal_ex(context, result, &md_len); EVP_MD_CTX_free(context); - char buf [EVP_MAX_MD_SIZE * 2] = {{0}}; + char buf [EVP_MAX_MD_SIZE * 2] = {}; for (unsigned int i = 0 ; i < md_len ; ++i){ sprintf(&buf[i*2], "%02x", result[i]); } From 5ddfac01f46f1041579675ec6ba15d7d5b852e9c Mon Sep 17 00:00:00 2001 From: Nicolas Josef Zunker Date: Tue, 13 Jun 2023 22:55:38 +0200 Subject: [PATCH 61/61] Small fixes + delete build.sh Remove an unnecessary #ifdef in Receiver.h, and fix error messages in AlcPacket.cpp --- build.sh | 6 ------ include/Receiver.h | 3 --- src/AlcPacket.cpp | 6 +++--- 3 files changed, 3 insertions(+), 12 deletions(-) delete mode 100755 build.sh diff --git a/build.sh b/build.sh deleted file mode 100755 index b3c4eab..0000000 --- a/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -rm -rf build -mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Debug -GNinja .. -ninja -j`nproc` diff --git a/include/Receiver.h b/include/Receiver.h index b01f52e..e9ef621 100644 --- a/include/Receiver.h +++ b/include/Receiver.h @@ -104,8 +104,5 @@ namespace LibFlute { completion_callback_t _completion_cb = nullptr; bool _running = true; -#ifdef SIMULATED_PKT_LOSS - unsigned int packets_dropped = 0; -#endif }; }; diff --git a/src/AlcPacket.cpp b/src/AlcPacket.cpp index 12e0002..3b959a2 100644 --- a/src/AlcPacket.cpp +++ b/src/AlcPacket.cpp @@ -88,7 +88,7 @@ LibFlute::AlcPacket::AlcPacket(char* data, size_t len) _fec_oti.encoding_id = FecScheme::Raptor; break; default: - throw "Only Compact No-Code FEC is supported"; + throw "Only the Compact No-Code and Raptor FEC schemes are supported"; break; } @@ -134,8 +134,8 @@ LibFlute::AlcPacket::AlcPacket(char* data, size_t len) break; case FecScheme::Raptor: //TODO - spdlog::warn("Raptor FEC support is still in progress"); - throw "Raptor FEC support is still in progress"; + spdlog::warn("Raptor FEC support in EXT_FTI header extension is still in progress"); + throw "Raptor FEC support in EXT_FTI header extension is still in progress"; break; default: throw "Unsupported FEC scheme";