Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduces CRCs #98

Merged
merged 4 commits into from
Apr 22, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/target
**/*.rs.bk
Cargo.lock
Cargo.lock
.vscode
9 changes: 9 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -52,13 +52,22 @@ path = "./postcard-derive"
version = "0.1.1"
optional = true

[dependencies.crc]
version = "3.0.1"
optional = true

[dependencies.paste]
version = "1.0.12"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For some help with our declarative macros.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it make sense to make this optional and gate it on the crc feature?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could, although perhaps that adds a little complexity in case we need it elsewhere?

Alternatively, let's get rid of the macros and the need for this. :-)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Commit 2c9ed55 makes the paste crate optional and includes it when use-crc is enabled.

optional = true

[features]
default = ["heapless-cas"]

use-std = ["serde/std", "alloc"]
heapless-cas = ["heapless", "heapless/cas"]
alloc = ["serde/alloc"]
use-defmt = ["defmt"]
use-crc = ["crc", "paste"]

# Experimental features!
#
215 changes: 117 additions & 98 deletions src/de/flavors.rs
Original file line number Diff line number Diff line change
@@ -163,120 +163,139 @@ impl<'de> Flavor<'de> for Slice<'de> {
}
}

// This is a terrible checksum implementation to make sure that we can effectively
// use the deserialization flavor. This is kept as a test (and not published)
// because an 8-bit checksum is not ACTUALLY useful for almost anything.
//
// You could certainly do something similar with a CRC32, cryptographic sig,
// or something else
#[cfg(test)]
mod test {
use super::*;
use serde::{Deserialize, Serialize};
////////////////////////////////////////
// CRC
////////////////////////////////////////

struct Checksum<'de, F>
where
F: Flavor<'de> + 'de,
{
flav: F,
checksum: u8,
_plt: PhantomData<&'de ()>,
}
/// This Cyclic Redundancy Check flavor applies [the CRC crate's `Algorithm`](https://docs.rs/crc/latest/crc/struct.Algorithm.html) struct on
/// the serialized data. The flavor will check the CRC assuming that it has been appended to the bytes.
///
/// CRCs are used for error detection when reading data back.
///
/// The `crc` feature requires enabling to use this module.
///
/// More on CRCs: <https://en.wikipedia.org/wiki/Cyclic_redundancy_check>.
#[cfg(feature = "use-crc")]
pub mod crc {
use core::convert::TryInto;

use crc::Digest;
use crc::Width;
use serde::Deserialize;

impl<'de, F> Checksum<'de, F>
use super::Flavor;
use super::Slice;

use crate::Deserializer;
use crate::Error;
use crate::Result;
use paste::paste;

/// Manages CRC modifications as a flavor.
pub struct CrcModifier<'de, B, W>
where
F: Flavor<'de> + 'de,
B: Flavor<'de>,
W: Width,
{
pub fn from_flav(flav: F) -> Self {
Self {
flav,
checksum: 0,
_plt: PhantomData,
}
}
flav: B,
digest: Digest<'de, W>,
}

impl<'de, F> Flavor<'de> for Checksum<'de, F>
impl<'de, B, W> CrcModifier<'de, B, W>
where
F: Flavor<'de> + 'de,
B: Flavor<'de>,
W: Width,
{
type Remainder = (<F as Flavor<'de>>::Remainder, u8);
type Source = F;

fn pop(&mut self) -> Result<u8> {
match self.flav.pop() {
Ok(u) => {
self.checksum = self.checksum.wrapping_add(u);
Ok(u)
}
Err(e) => Err(e),
}
}
fn try_take_n(&mut self, ct: usize) -> Result<&'de [u8]> {
match self.flav.try_take_n(ct) {
Ok(u) => {
u.iter().for_each(|u| {
self.checksum = self.checksum.wrapping_add(*u);
});
Ok(u)
}
Err(e) => Err(e),
}
}
fn finalize(self) -> Result<Self::Remainder> {
Ok((self.flav.finalize()?, self.checksum))
/// Create a new Crc modifier Flavor.
pub fn new(bee: B, digest: Digest<'de, W>) -> Self {
Self { flav: bee, digest }
}
}

#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
pub struct SomeData<'a> {
#[serde(borrow)]
sli: &'a [u8],
sts: &'a str,
foo: u64,
bar: u128,
}
macro_rules! impl_flavor {
($( $int:ty ),*) => {
$(
paste! {
impl<'de, B> Flavor<'de> for CrcModifier<'de, B, $int>
where
B: Flavor<'de>,
{
type Remainder = B::Remainder;

#[test]
fn smoke() {
const EXPECTED: &[u8] = &[
4, 255, 1, 34, 51, 19, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 103, 111, 111,
100, 32, 116, 101, 115, 116, 170, 213, 170, 213, 170, 213, 170, 213, 170, 1, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127,
];
type Source = B::Source;

// Calculate simple 8-bit checksum
let mut check: u8 = 0;
EXPECTED.iter().for_each(|u| check = check.wrapping_add(*u));
#[inline]
fn pop(&mut self) -> Result<u8> {
match self.flav.pop() {
Ok(byte) => {
self.digest.update(&[byte]);
Ok(byte)
}
e @ Err(_) => e,
}
}

let mut buf = [0u8; 256];
let data = SomeData {
sli: &[0xFF, 0x01, 0x22, 0x33],
sts: "this is a good test",
foo: (u64::MAX / 3) * 2,
bar: u128::MAX / 4,
};
let used = crate::to_slice(&data, &mut buf).unwrap();
assert_eq!(used, EXPECTED);
let used = used.len();
#[inline]
fn try_take_n(&mut self, ct: usize) -> Result<&'de [u8]> {
match self.flav.try_take_n(ct) {
Ok(bytes) => {
self.digest.update(bytes);
Ok(bytes)
}
e @ Err(_) => e,
}
}

// Put the checksum at the end
buf[used] = check;
fn finalize(mut self) -> Result<Self::Remainder> {
match self.flav.try_take_n(core::mem::size_of::<$int>()) {
Ok(prev_crc_bytes) => match self.flav.finalize() {
Ok(remainder) => {
let crc = self.digest.finalize();
let le_bytes = prev_crc_bytes
.try_into()
.map_err(|_| Error::DeserializeBadEncoding)?;
let prev_crc = <$int>::from_le_bytes(le_bytes);
if crc == prev_crc {
Ok(remainder)
} else {
Err(Error::DeserializeBadEncoding)
}
}
e @ Err(_) => e,
},
Err(e) => Err(e),
}
}
}

let mut deser = crate::de::Deserializer::from_flavor(Checksum::from_flav(Slice::new(&buf)));
/// Deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any)
/// of the byte slice is not returned.
pub fn [<from_bytes_ $int>]<'a, T>(s: &'a [u8], digest: Digest<'a, $int>) -> Result<T>
where
T: Deserialize<'a>,
{
let flav = CrcModifier::new(Slice::new(s), digest);
let mut deserializer = Deserializer::from_flavor(flav);
let r = T::deserialize(&mut deserializer)?;
let _ = deserializer.finalize()?;
Ok(r)
}

let t = SomeData::<'_>::deserialize(&mut deser).unwrap();
assert_eq!(t, data);

// Normally, you'd probably expect the check
let (rem, cksm) = deser.finalize().unwrap();

// The pre-calculated checksum we stuffed at the end is the
// first "unused" byte
assert_eq!(rem[0], check);

// the one we calculated during serialization matches the
// pre-calculated one
assert_eq!(cksm, check);
/// Deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any)
/// of the byte slice is returned for further usage
pub fn [<take_from_bytes_ $int>]<'a, T>(s: &'a [u8], digest: Digest<'a, $int>) -> Result<(T, &'a [u8])>
where
T: Deserialize<'a>,
{
let flav = CrcModifier::new(Slice::new(s), digest);
let mut deserializer = Deserializer::from_flavor(flav);
let t = T::deserialize(&mut deserializer)?;
Ok((t, deserializer.finalize()?))
}
}
)*
};
}

impl_flavor![u8, u16, u32, u64, u128];
}
20 changes: 18 additions & 2 deletions src/de/mod.rs
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@ where

/// Deserialize a message of type `T` from a cobs-encoded byte slice. The
/// unused portion (if any) of the byte slice is not returned.
/// The used portion of the input slice is modified during deserialization (even if an error is returned).
/// The used portion of the input slice is modified during deserialization (even if an error is returned).
/// Therefore, if this is not desired, pass a clone of the original slice.
pub fn from_bytes_cobs<'a, T>(s: &'a mut [u8]) -> Result<T>
where
@@ -32,7 +32,7 @@ where

/// Deserialize a message of type `T` from a cobs-encoded byte slice. The
/// unused portion (if any) of the byte slice is returned for further usage.
/// The used portion of the input slice is modified during deserialization (even if an error is returned).
/// The used portion of the input slice is modified during deserialization (even if an error is returned).
/// Therefore, if this is not desired, pass a clone of the original slice.
pub fn take_from_bytes_cobs<'a, T>(s: &'a mut [u8]) -> Result<(T, &'a mut [u8])>
where
@@ -67,6 +67,22 @@ where
Ok((t, deserializer.finalize()?))
}

/// Conveniently deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any)
/// of the byte slice is not returned.
///
/// See the `de_flavors::crc` module for the complete set of functions.
///
#[cfg(feature = "use-crc")]
pub use flavors::crc::from_bytes_u32 as from_bytes_crc32;

/// Conveniently deserialize a message of type `T` from a byte slice with a Crc. The unused portion (if any)
/// of the byte slice is returned for further usage
///
/// See the `de_flavors::crc` module for the complete set of functions.
///
#[cfg(feature = "use-crc")]
pub use flavors::crc::take_from_bytes_u32 as take_from_bytes_crc32;

jamesmunns marked this conversation as resolved.
Show resolved Hide resolved
////////////////////////////////////////////////////////////////////////////////

#[cfg(feature = "heapless")]
15 changes: 15 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
@@ -91,6 +91,21 @@ pub use ser::{to_stdvec, to_stdvec_cobs};
#[cfg(feature = "alloc")]
pub use ser::{to_allocvec, to_allocvec_cobs};

#[cfg(feature = "use-crc")]
pub use {
de::{from_bytes_crc32, take_from_bytes_crc32},
ser::to_slice_crc32,
};

#[cfg(all(feature = "use-crc", feature = "heapless"))]
pub use ser::to_vec_crc32;

#[cfg(all(feature = "use-crc", feature = "use-std"))]
pub use ser::to_stdvec_crc32;

#[cfg(all(feature = "use-crc", feature = "alloc"))]
pub use ser::to_allocvec_crc32;

jamesmunns marked this conversation as resolved.
Show resolved Hide resolved
#[cfg(test)]
mod test {
#[test]
131 changes: 129 additions & 2 deletions src/ser/flavors.rs
Original file line number Diff line number Diff line change
@@ -101,6 +101,9 @@ pub use std_vec::*;
#[cfg(feature = "alloc")]
pub use alloc_vec::*;

#[cfg(feature = "alloc")]
extern crate alloc;

/// The serialization Flavor trait
///
/// This is used as the primary way to encode serialized data into some kind of buffer,
@@ -236,7 +239,7 @@ mod heapless_vec {
}
}

impl<'a, const B: usize> Flavor for HVec<B> {
jamesmunns marked this conversation as resolved.
Show resolved Hide resolved
impl<const B: usize> Flavor for HVec<B> {
type Output = Vec<u8, B>;

#[inline(always)]
@@ -380,7 +383,7 @@ where
}
}

impl<'a, B> Flavor for Cobs<B>
jamesmunns marked this conversation as resolved.
Show resolved Hide resolved
impl<B> Flavor for Cobs<B>
where
B: Flavor + IndexMut<usize, Output = u8>,
{
@@ -411,6 +414,130 @@ where
}
}

////////////////////////////////////////
// CRC
////////////////////////////////////////

/// This Cyclic Redundancy Check flavor applies [the CRC crate's `Algorithm`](https://docs.rs/crc/latest/crc/struct.Algorithm.html) struct on
/// the serialized data. The output of this flavor receives the CRC appended to the bytes.
///
/// CRCs are used for error detection when reading data back.
///
/// The `crc` feature requires enabling to use this module.
///
/// More on CRCs: <https://en.wikipedia.org/wiki/Cyclic_redundancy_check>.
#[cfg(feature = "use-crc")]
pub mod crc {
use crc::Digest;
use crc::Width;
use serde::Serialize;

#[cfg(feature = "alloc")]
use super::alloc;
use super::Flavor;
use super::Slice;

use crate::serialize_with_flavor;
use crate::Result;
use paste::paste;

/// Manages CRC modifications as a flavor.
pub struct CrcModifier<'a, B, W>
where
B: Flavor,
W: Width,
{
flav: B,
digest: Digest<'a, W>,
}

impl<'a, B, W> CrcModifier<'a, B, W>
where
B: Flavor,
W: Width,
{
/// Create a new CRC modifier Flavor.
pub fn new(bee: B, digest: Digest<'a, W>) -> Self {
Self { flav: bee, digest }
}
}

macro_rules! impl_flavor {
($( $int:ty ),*) => {
$(
paste! {
impl<'a, B> Flavor for CrcModifier<'a, B, $int>
where
B: Flavor,
{
type Output = <B as Flavor>::Output;

#[inline(always)]
fn try_push(&mut self, data: u8) -> Result<()> {
self.digest.update(&[data]);
self.flav.try_push(data)
}

fn finalize(mut self) -> Result<Self::Output> {
let crc = self.digest.finalize();
for byte in crc.to_le_bytes() {
self.flav.try_push(byte)?;
}
self.flav.finalize()
}
}

/// Serialize a `T` to the given slice, with the resulting slice containing
/// data followed by a CRC. The CRC bytes are included in the output buffer.
///
/// When successful, this function returns the slice containing the
/// serialized and encoded message.
pub fn [<to_slice_ $int>]<'a, 'b, T>(
value: &'b T,
buf: &'a mut [u8],
digest: Digest<'a, $int>,
) -> Result<&'a mut [u8]>
where
T: Serialize + ?Sized,
{
serialize_with_flavor(value, CrcModifier::new(Slice::new(buf), digest))
}

/// Serialize a `T` to a `heapless::Vec<u8>`, with the `Vec` containing
/// data followed by a CRC. The CRC bytes are included in the output `Vec`.
/// Requires the (default) `heapless` feature.
#[cfg(feature = "heapless")]
pub fn [<to_vec_ $int>]<'a, T, const B: usize>(
value: &T,
digest: Digest<'a, $int>,
) -> Result<heapless::Vec<u8, B>>
where
T: Serialize + ?Sized,
{
use super::HVec;

serialize_with_flavor(value, CrcModifier::new(HVec::default(), digest))
}

/// Serialize a `T` to a `heapless::Vec<u8>`, with the `Vec` containing
/// data followed by a CRC. The CRC bytes are included in the output `Vec`.
#[cfg(feature = "alloc")]
pub fn [<to_allocvec_ $int>]<'a, T>(value: &T, digest: Digest<'a, $int>) -> Result<alloc::vec::Vec<u8>>
where
T: Serialize + ?Sized,
{
use super::AllocVec;

serialize_with_flavor(value, CrcModifier::new(AllocVec::new(), digest))
}
}
)*
};
}

impl_flavor![u8, u16, u32, u64, u128];
}

/// The `Size` flavor is a measurement flavor, which accumulates the number of bytes needed to
/// serialize the data.
///
101 changes: 101 additions & 0 deletions src/ser/mod.rs
Original file line number Diff line number Diff line change
@@ -237,6 +237,107 @@ where
)
}

/// Conveniently serialize a `T` to the given slice, with the resulting slice containing
/// data followed by a 32-bit CRC. The CRC bytes are included in the output buffer.
///
/// When successful, this function returns the slice containing the
/// serialized and encoded message.
///
/// ## Example
///
/// ```rust
/// use crc::{Crc, CRC_32_ISCSI};
///
/// let mut buf = [0; 9];
///
/// let data: &[u8] = &[0x01, 0x00, 0x20, 0x30];
/// let crc = Crc::<u32>::new(&CRC_32_ISCSI);
/// let used = postcard::to_slice_crc32(data, &mut buf, crc.digest()).unwrap();
/// assert_eq!(used, &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]);
/// ```
///
/// See the `ser_flavors::crc` module for the complete set of functions.
///
#[cfg(feature = "use-crc")]
pub use flavors::crc::to_slice_u32 as to_slice_crc32;

/// Conveniently serialize a `T` to a `heapless::Vec<u8>`, with the `Vec` containing
/// data followed by a 32-bit CRC. The CRC bytes are included in the output `Vec`.
/// Requires the (default) `heapless` feature.
///
/// ## Example
///
/// ```rust
/// use crc::{Crc, CRC_32_ISCSI};
/// use heapless::Vec;
/// use core::ops::Deref;
///
/// // NOTE: postcard handles `&[u8]` and `&[u8; N]` differently.
/// let data: &[u8] = &[0x01u8, 0x00, 0x20, 0x30];
/// let crc = Crc::<u32>::new(&CRC_32_ISCSI);
/// let ser: Vec<u8, 32> = postcard::to_vec_crc32(data, crc.digest()).unwrap();
/// assert_eq!(ser.deref(), &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]);
///
/// let data: &[u8; 4] = &[0x01u8, 0x00, 0x20, 0x30];
/// let ser: Vec<u8, 32> = postcard::to_vec_crc32(data, crc.digest()).unwrap();
/// assert_eq!(ser.deref(), &[0x01, 0x00, 0x20, 0x30, 0xCC, 0x4B, 0x4A, 0xDA]);
/// ```
///
/// See the `ser_flavors::crc` module for the complete set of functions.
///
#[cfg(all(feature = "use-crc", feature = "heapless"))]
pub use flavors::crc::to_vec_u32 as to_vec_crc32;

/// Conveniently serialize a `T` to a `heapless::Vec<u8>`, with the `Vec` containing
/// data followed by a 32-bit CRC. The CRC bytes are included in the output `Vec`.
///
/// ## Example
///
/// ```rust
/// use crc::{Crc, CRC_32_ISCSI};
/// use core::ops::Deref;
///
/// // NOTE: postcard handles `&[u8]` and `&[u8; N]` differently.
/// let data: &[u8] = &[0x01u8, 0x00, 0x20, 0x30];
/// let crc = Crc::<u32>::new(&CRC_32_ISCSI);
/// let ser: Vec<u8> = postcard::to_stdvec_crc32(data, crc.digest()).unwrap();
/// assert_eq!(ser.deref(), &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]);
///
/// let data: &[u8; 4] = &[0x01u8, 0x00, 0x20, 0x30];
/// let ser: Vec<u8> = postcard::to_stdvec_crc32(data, crc.digest()).unwrap();
/// assert_eq!(ser.deref(), &[0x01, 0x00, 0x20, 0x30, 0xCC, 0x4B, 0x4A, 0xDA]);
/// ```
///
/// See the `ser_flavors::crc` module for the complete set of functions.
///
#[cfg(all(feature = "use-crc", feature = "use-std"))]
pub use flavors::crc::to_allocvec_u32 as to_stdvec_crc32;

/// Conveniently serialize a `T` to a `heapless::Vec<u8>`, with the `Vec` containing
/// data followed by a 32-bit CRC. The CRC bytes are included in the output `Vec`.
///
/// ## Example
///
/// ```rust
/// use crc::{Crc, CRC_32_ISCSI};
/// use core::ops::Deref;
///
/// // NOTE: postcard handles `&[u8]` and `&[u8; N]` differently.
/// let data: &[u8] = &[0x01u8, 0x00, 0x20, 0x30];
/// let crc = Crc::<u32>::new(&CRC_32_ISCSI);
/// let ser: Vec<u8> = postcard::to_allocvec_crc32(data, crc.digest()).unwrap();
/// assert_eq!(ser.deref(), &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]);
///
/// let data: &[u8; 4] = &[0x01u8, 0x00, 0x20, 0x30];
/// let ser: Vec<u8> = postcard::to_allocvec_crc32(data, crc.digest()).unwrap();
/// assert_eq!(ser.deref(), &[0x01, 0x00, 0x20, 0x30, 0xCC, 0x4B, 0x4A, 0xDA]);
/// ```
///
/// See the `ser_flavors::crc` module for the complete set of functions.
///
#[cfg(all(feature = "use-crc", feature = "alloc"))]
pub use flavors::crc::to_allocvec_u32 as to_allocvec_crc32;

/// `serialize_with_flavor()` has three generic parameters, `T, F, O`.
///
/// * `T`: This is the type that is being serialized
39 changes: 39 additions & 0 deletions tests/crc.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#[test]
#[cfg(feature = "use-crc")]
fn test_crc() {
use crc::{Crc, CRC_32_ISCSI};

let data: &[u8] = &[0x01, 0x00, 0x20, 0x30];
let buffer = &mut [0u8; 32];
let crc = Crc::<u32>::new(&CRC_32_ISCSI);
let digest = crc.digest();
let res = postcard::to_slice_crc32(data, buffer, digest).unwrap();
assert_eq!(res, &[0x04, 0x01, 0x00, 0x20, 0x30, 0x8E, 0xC8, 0x1A, 0x37]);

let digest = crc.digest();
let res = postcard::take_from_bytes_crc32::<[u8; 5]>(&res, digest).unwrap();

let expected_bytes = [0x04, 0x01, 0x00, 0x20, 0x30];
let remaining_bytes = [];
assert_eq!(res, (expected_bytes, remaining_bytes.as_slice()));
}

#[test]
#[cfg(feature = "use-crc")]
fn test_crc_8() {
use crc::{Crc, CRC_8_SMBUS};

let data: &[u8] = &[0x01, 0x00, 0x20, 0x30];
let buffer = &mut [0u8; 32];
let crc = Crc::<u8>::new(&CRC_8_SMBUS);
let digest = crc.digest();
let res = postcard::ser_flavors::crc::to_slice_u8(data, buffer, digest).unwrap();
assert_eq!(res, &[0x04, 0x01, 0x00, 0x20, 0x30, 167]);

let digest = crc.digest();
let res = postcard::de_flavors::crc::take_from_bytes_u8::<[u8; 5]>(&res, digest).unwrap();

let expected_bytes = [0x04, 0x01, 0x00, 0x20, 0x30];
let remaining_bytes = [];
assert_eq!(res, (expected_bytes, remaining_bytes.as_slice()));
}