Add support form parsing dlt-messages from streams (esrlabs#34).

Cleanup of feature names.

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.20.0] - 2025-02-11
+
+### Added
+
+- Support parsing of DLT messages from streams
+
+### Changed
+
+- Cleanup feature names
+
 ## [0.19.2] - 2025-02-06
 
 ### Changed
@@ -36,7 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Add feature "serde-support", which adds to crate's types Serialize/Deserialize
+- Add feature "serialization", which adds to crate's types Serialize/Deserialize
 
 ## [0.17.0] - 2024-10-04
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dlt-core"
-version = "0.19.2"
+version = "0.20.0"
 authors = ["esrlabs.com"]
 edition = "2021"
 description = """
@@ -21,13 +21,25 @@ rustc-hash = { version = "2.1", optional = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
 thiserror = "1.0"
+futures = "0.3"
+tokio = { version = "1", features = ["full"], optional = true }
+tokio-stream = { version = "0.1", optional = true }
+tokio-util = { version = "0.7", optional = true }
 
 [features]
 default = []
 statistics = ["buf_redux", "rustc-hash"]
-fibex_parser = ["quick-xml"]
-debug_parser = []
-serde-support = ["serde", "serde_json"]
+fibex = ["quick-xml"]
+debug = []
+serialization = [
+    "serde",
+    "serde_json"
+]
+stream = [ 
+    "tokio", 
+    "tokio-stream", 
+    "tokio-util" 
+]
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] }

README.md

@@ -150,11 +150,13 @@ Below is the revised and improved English version of the documentation:
 
 * **`statistics`**: Enables the `statistics` module, which scans the source data and provides a summary of its contents. This gives you an overview of the number of messages and their content.
 
-* **`fibex_parser`**: Enables the `fibex` module, which allows to parse configurations for non-verbose messages from a fibex model.
+* **`fibex`**: Enables the `fibex` module, which allows to parse configurations for non-verbose messages from a fibex model.
 
-* **`debug_parser`**: Adds additional log output for debugging purposes.
+* **`debug`**: Adds additional log output for debugging purposes.
 
-* **`serde-support`**: Adds `Serialize` and `Deserialize` implementations (via `serde`) to all public types. This feature is useful if you need to encode or decode these types for transmission or storage.
+* **`serialization`**: Adds `Serialize` and `Deserialize` implementations (via `serde`) to all public types. This feature is useful if you need to encode or decode these types for transmission or storage.
+
+* **`stream`**: Provides API for parsing DLT messages from streams.
 
 ## Example users
 

src/dlt.rs

@@ -42,7 +42,7 @@ pub enum Error {
 
 /// Used to express the byte order of DLT data type fields
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, PartialOrd, Ord)]
@@ -56,7 +56,7 @@ pub enum Endianness {
 
 /// represents a DLT message including all headers
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -69,7 +69,7 @@ pub struct Message {
 
 /// Storage header is used in case of dlt entries stored in file
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -82,7 +82,7 @@ pub struct StorageHeader {
 
 /// The Standard Header shall be in big endian format
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -107,7 +107,7 @@ pub struct StandardHeader {
 ///
 /// The Extended Header shall be in big endian format
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -144,7 +144,7 @@ pub struct ExtendedHeader {
 /// and payload. The payload contains of the Service ID and the contained parameters.
 ///
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -177,7 +177,7 @@ pub enum PayloadContent {
 }
 
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -343,7 +343,7 @@ impl StandardHeader {
 
 /// Representation of log levels used in DLT log messages
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, PartialOrd, Clone, Copy)]
@@ -378,7 +378,7 @@ impl AsRef<str> for LogLevel {
 /// In case the dlt message contains tracing information, the Trace-Type
 /// indicates different kinds of trace message types
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -414,7 +414,7 @@ impl AsRef<str> for ApplicationTraceType {
 /// In case the dlt message contains networking information,
 /// the Trace-Type indicates different kinds of network message types
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -457,7 +457,7 @@ const CTRL_TYPE_RESPONSE: u8 = 0x2;
 /// In case the dlt message contains control information,
 /// the Trace-Type indicates different kinds of control message types
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -498,7 +498,7 @@ impl ControlType {
 
 /// Part of the extended header, distinguishes log, trace and controll messages
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -559,7 +559,7 @@ impl ExtendedHeader {
 /// Fixed-Point representation. only supports 32 bit and 64 bit values
 /// according to the spec 128 bit are possible but we don't support it
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -578,7 +578,7 @@ pub(crate) fn fixed_point_value_width(v: &FixedPointValue) -> usize {
 
 /// Represents the value of an DLT argument
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, PartialEq, Clone)]
@@ -603,7 +603,7 @@ pub enum Value {
 /// Defines what string type is used, `ASCII` or `UTF8`
 #[allow(clippy::upper_case_acronyms)]
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -620,7 +620,7 @@ pub enum StringCoding {
 
 /// Represents the bit width of a floatingpoint value type
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq, Copy)]
@@ -639,7 +639,7 @@ pub(crate) fn float_width_to_type_length(width: FloatWidth) -> TypeLength {
 
 /// Represents the bit width of a value type
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq, Copy)]
@@ -677,7 +677,7 @@ impl TypeLength {
 ///
 /// the Array type is not yet supported and honestly I never saw anyone using it
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -724,7 +724,7 @@ pub enum TypeInfoKind {
 /// number of characters of the associated name or unit filed. The unit
 /// information is to add only in some data types.
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -910,7 +910,7 @@ impl TryFrom<u32> for TypeInfo {
 ///     * i64 bit if Type Length (TYLE) equals 4
 ///     * i128 bit if Type Length (TYLE) equals 5 (unsupported)
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -927,7 +927,7 @@ pub struct FixedPoint {
 /// itself, it is needed to provide information like size and type
 /// of the variable. This information is contained in the `type_info` field.
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -1528,7 +1528,7 @@ fn payload_content_len(content: &PayloadContent) -> usize {
 
 /// Configuration options for the extended header
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -1540,7 +1540,7 @@ pub struct ExtendedHeaderConfig {
 
 /// Configuration options for a DLT message, used when constructing a message
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone, PartialEq)]
@@ -1557,7 +1557,7 @@ pub struct MessageConfig {
 
 #[inline]
 fn dbg_bytes_with_info(_name: &str, _bytes: &[u8], _info: Option<&str>) {
-    #[cfg(feature = "debug_parser")]
+    #[cfg(feature = "debug")]
     {
         trace!(
             "writing {}: {} {:02X?} {}",
@@ -1708,6 +1708,8 @@ pub(crate) const WITH_ECU_ID_FLAG: u8 = 1 << 2;
 pub(crate) const WITH_SESSION_ID_FLAG: u8 = 1 << 3;
 pub(crate) const WITH_TIMESTAMP_FLAG: u8 = 1 << 4;
 pub(crate) const HEADER_MIN_LENGTH: u16 = 4;
+#[cfg(feature = "stream")]
+pub(crate) const HEADER_MAX_LENGTH: u16 = 16;
 
 // Verbose Mode
 

src/fibex/mod.rs

@@ -55,7 +55,7 @@ pub enum Error {
 
 /// Contains all the paths of fibex files that should be combined into the model
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug)]

src/filtering.rs

@@ -25,7 +25,7 @@ use std::{collections::HashSet, iter::FromIterator};
 /// only this is possible:
 /// - `app-id is_one_of ["abc","foo"] AND log-level <= DEBUG`
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Clone)]
@@ -96,8 +96,8 @@ impl From<&DltFilterConfig> for ProcessedDltFilterConfig {
     }
 }
 
-/// Read filter config from a json file. Available only with feature "serde-support"
-#[cfg(feature = "serde-support")]
+/// Read filter config from a json file. Available only with feature "serialization"
+#[cfg(feature = "serialization")]
 pub fn read_filter_options(f: &mut std::fs::File) -> Option<DltFilterConfig> {
     use std::io::Read;
 

src/lib.rs

@@ -20,7 +20,7 @@
 extern crate log;
 
 pub mod dlt;
-#[cfg(feature = "fibex_parser")]
+#[cfg(feature = "fibex")]
 pub mod fibex;
 pub mod filtering;
 pub mod parse;
@@ -30,6 +30,9 @@ pub mod service_id;
 #[cfg(not(tarpaulin_include))]
 #[cfg(feature = "statistics")]
 pub mod statistics;
+#[cfg(not(tarpaulin_include))]
+#[cfg(feature = "stream")]
+pub mod stream;
 
 #[cfg(test)]
 pub mod proptest_strategies;

src/parse.rs

@@ -747,7 +747,7 @@ fn dlt_payload<T: NomByteOrder>(
 
 #[inline]
 fn dbg_parsed<T: std::fmt::Debug>(_name: &str, _before: &[u8], _after: &[u8], _value: &T) {
-    // #[cfg(feature = "debug_parser")]
+    // #[cfg(feature = "debug")]
     {
         let input_len = _before.len();
         let now_len = _after.len();

src/statistics.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! # rapidly gather statistics info of a dlt source
+//! # Rapidly gather statistics info of a dlt source
 use crate::{
     dlt::{LogLevel, MessageType},
     parse::{
@@ -93,7 +93,7 @@ pub fn dlt_statistic_row_info(
 
 /// Shows how many messages per log level where found
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug, Default, Clone)]
@@ -164,7 +164,7 @@ type IdMap = FxHashMap<String, LevelDistribution>;
 /// Includes the `LevelDistribution` for all `app-ids`, `context-ids` and
 /// `ecu_ids`
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug)]
@@ -215,7 +215,7 @@ impl Default for StatisticInfo {
 
 /// Stats about a row in a DLT file
 #[cfg_attr(
-    feature = "serde-support",
+    feature = "serialization",
     derive(serde::Serialize, serde::Deserialize)
 )]
 #[derive(Debug)]