Skip to content

Commit

Permalink
Add v3-preview module gathering a few breaking changes (#105)
Browse files Browse the repository at this point in the history
This new module addresses the following breaking changes of #72:
- `static` instead of `const` for pre-defined encodings
- Private `Encoding` implementation with `unsafe` constructor
- Use `MaybeUninit` internally and expose `_uninit` functions
- Use static dispatch instead of dynamic dispatch (fixing #97)
  • Loading branch information
ia0 authored May 5, 2024
1 parent 18203b9 commit 2591b46
Show file tree
Hide file tree
Showing 11 changed files with 1,556 additions and 90 deletions.
1 change: 1 addition & 0 deletions lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ rustdoc-args = ["--cfg=docsrs"]
default = ["std"]
alloc = []
std = ["alloc"]
v3-preview = []
52 changes: 38 additions & 14 deletions lib/benches/lib.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
#![feature(test)]

extern crate data_encoding;
extern crate test;

use data_encoding::{Specification, BASE32, BASE32_DNSCURVE, BASE64, BASE64_NOPAD, HEXLOWER};
#[cfg(feature = "v3-preview")]
use std::convert::TryFrom as _;

#[cfg(not(feature = "v3-preview"))]
use data_encoding as constants;
#[cfg(feature = "v3-preview")]
use data_encoding::v3_preview as constants;
#[cfg(feature = "v3-preview")]
use data_encoding::v3_preview::{Bit1, Bit2, Bit3, Bit6, Encoding, False, True};
use data_encoding::Specification;
use test::Bencher;

#[bench]
Expand All @@ -13,6 +21,8 @@ fn base02_encode_base(b: &mut Bencher) {
let mut spec = Specification::new();
spec.symbols.push_str("01");
let base = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base = Encoding::<Bit1, True, False, False, False>::try_from(base).unwrap();
b.iter(|| base.encode_mut(input, output));
}

Expand All @@ -23,6 +33,8 @@ fn base02_decode_base(b: &mut Bencher) {
let mut spec = Specification::new();
spec.symbols.push_str("01");
let base = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base = Encoding::<Bit1, True, False, False, False>::try_from(base).unwrap();
b.iter(|| base.decode_mut(input, output));
}

Expand All @@ -33,6 +45,8 @@ fn base04_encode_base(b: &mut Bencher) {
let mut spec = Specification::new();
spec.symbols.push_str("0123");
let base = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base = Encoding::<Bit2, True, False, False, False>::try_from(base).unwrap();
b.iter(|| base.encode_mut(input, output));
}

Expand All @@ -43,6 +57,8 @@ fn base04_decode_base(b: &mut Bencher) {
let mut spec = Specification::new();
spec.symbols.push_str("0123");
let base = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base = Encoding::<Bit2, True, False, False, False>::try_from(base).unwrap();
b.iter(|| base.decode_mut(input, output));
}

Expand All @@ -53,6 +69,8 @@ fn base08_encode_base(b: &mut Bencher) {
let mut spec = Specification::new();
spec.symbols.push_str("01234567");
let base = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base = Encoding::<Bit3, True, False, False, False>::try_from(base).unwrap();
b.iter(|| base.encode_mut(input, output));
}

Expand All @@ -63,56 +81,58 @@ fn base08_decode_base(b: &mut Bencher) {
let mut spec = Specification::new();
spec.symbols.push_str("01234567");
let base = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base = Encoding::<Bit3, True, False, False, False>::try_from(base).unwrap();
b.iter(|| base.decode_mut(input, output));
}

#[bench]
fn base16_encode_base(b: &mut Bencher) {
let input = &[0u8; 4096];
let output = &mut [0u8; 8192];
b.iter(|| HEXLOWER.encode_mut(input, output));
b.iter(|| constants::HEXLOWER.encode_mut(input, output));
}

#[bench]
fn base16_decode_base(b: &mut Bencher) {
let input = &[b'0'; 4096];
let output = &mut [0u8; 2048];
b.iter(|| HEXLOWER.decode_mut(input, output));
b.iter(|| constants::HEXLOWER.decode_mut(input, output));
}

#[bench]
fn base32_encode_base(b: &mut Bencher) {
let input = &[0u8; 4096];
let output = &mut [0u8; 6560];
b.iter(|| BASE32.encode_mut(input, output));
b.iter(|| constants::BASE32.encode_mut(input, output));
}

#[bench]
fn base32_decode_base(b: &mut Bencher) {
let input = &[b'A'; 4096];
let output = &mut [0u8; 2560];
b.iter(|| BASE32.decode_mut(input, output));
b.iter(|| constants::BASE32.decode_mut(input, output));
}

#[bench]
fn base64_encode_base(b: &mut Bencher) {
let input = &[0u8; 4096];
let output = &mut [0u8; 5462];
b.iter(|| BASE64_NOPAD.encode_mut(input, output));
b.iter(|| constants::BASE64_NOPAD.encode_mut(input, output));
}

#[bench]
fn base64_decode_base(b: &mut Bencher) {
let input = &[b'A'; 4096];
let output = &mut [0u8; 3072];
b.iter(|| BASE64_NOPAD.decode_mut(input, output));
b.iter(|| constants::BASE64_NOPAD.decode_mut(input, output));
}

#[bench]
fn base64_encode_pad(b: &mut Bencher) {
let input = &mut [b'A'; 4096];
let output = &mut [0u8; 5464];
b.iter(|| BASE64.encode_mut(input, output));
b.iter(|| constants::BASE64.encode_mut(input, output));
}

#[bench]
Expand All @@ -126,17 +146,19 @@ fn base64_decode_pad(b: &mut Bencher) {
}
}
let output = &mut [0u8; 3072];
b.iter(|| BASE64.decode_mut(input, output).unwrap());
b.iter(|| constants::BASE64.decode_mut(input, output).unwrap());
}

#[bench]
fn base64_encode_wrap(b: &mut Bencher) {
let input = &[0u8; 4096];
let output = &mut [0u8; 5608];
let mut spec = BASE64.specification();
let mut spec = data_encoding::BASE64.specification();
spec.wrap.width = 76;
spec.wrap.separator.push_str("\r\n");
let base64 = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base64 = Encoding::<Bit6, True, True, True, True>::try_from(base64).unwrap();
b.iter(|| base64.encode_mut(input, output));
}

Expand All @@ -148,23 +170,25 @@ fn base64_decode_wrap(b: &mut Bencher) {
input[x + 3] = b'\n';
}
let output = &mut [0u8; 3072];
let mut spec = BASE64.specification();
let mut spec = data_encoding::BASE64.specification();
spec.wrap.width = 76;
spec.wrap.separator.push_str("\r\n");
let base64 = spec.encoding().unwrap();
#[cfg(feature = "v3-preview")]
let base64 = Encoding::<Bit6, True, True, True, True>::try_from(base64).unwrap();
b.iter(|| base64.decode_mut(input, output).unwrap());
}

#[bench]
fn dnscurve_decode_base(b: &mut Bencher) {
let input = &[b'0'; 4096];
let output = &mut [0u8; 2560];
b.iter(|| BASE32_DNSCURVE.decode_mut(input, output));
b.iter(|| constants::BASE32_DNSCURVE.decode_mut(input, output));
}

#[bench]
fn dnscurve_encode_base(b: &mut Bencher) {
let input = &[0u8; 4096];
let output = &mut [0u8; 6554];
b.iter(|| BASE32_DNSCURVE.encode_mut(input, output));
b.iter(|| constants::BASE32_DNSCURVE.encode_mut(input, output));
}
7 changes: 7 additions & 0 deletions lib/fuzz/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,10 @@ name = "encode_write"
path = "fuzz_targets/encode_write.rs"
test = false
doc = false

[[bin]]
name = "v3-preview"
path = "fuzz_targets/v3-preview.rs"
test = false
doc = false
required-features = ["data-encoding/v3-preview"]
3 changes: 0 additions & 3 deletions lib/fuzz/examples/analyze.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
extern crate data_encoding;
extern crate data_encoding_fuzz;

use std::collections::HashMap;
use std::ops::AddAssign;

Expand Down
2 changes: 0 additions & 2 deletions lib/fuzz/examples/debug.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
extern crate data_encoding_fuzz;

use std::io::Read;

use data_encoding_fuzz::generate_specification;
Expand Down
98 changes: 98 additions & 0 deletions lib/fuzz/fuzz_targets/v3-preview.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#![no_main]

use data_encoding::v3_preview::{Bit1, Bit2, Bit3, Bit4, Bit5, Bit6, Encoding, False, True};
use data_encoding_fuzz::{decode_prefix, generate_encoding};
use libfuzzer_sys::fuzz_target;

fuzz_target!(|data: &[u8]| {
let mut data = data;
let dyn_base = generate_encoding(&mut data);
let mut count = 0;
macro_rules! test {
($Bit:ident, $Msb:ident, $Pad:ident, $Wrap:ident, $Ignore:ident) => {
if let Ok(base) = <&Encoding<$Bit, $Msb, $Pad, $Wrap, $Ignore>>::try_from(&dyn_base) {
count += 1;
let encoded = base.encode(data);
assert_eq!(encoded, dyn_base.encode(data));
assert_eq!(base.decode(encoded.as_bytes()).unwrap(), data);
if dyn_base.is_canonical() {
let raw = decode_prefix(&dyn_base, &mut data);
assert_eq!(base.encode(&raw).as_bytes(), data);
}
}
};
}
test!(Bit1, False, False, False, False);
test!(Bit1, False, False, False, True);
test!(Bit1, False, False, True, True);
test!(Bit1, False, True, False, False);
test!(Bit1, False, True, False, True);
test!(Bit1, False, True, True, True);
test!(Bit1, True, False, False, False);
test!(Bit1, True, False, False, True);
test!(Bit1, True, False, True, True);
test!(Bit1, True, True, False, False);
test!(Bit1, True, True, False, True);
test!(Bit1, True, True, True, True);
test!(Bit2, False, False, False, False);
test!(Bit2, False, False, False, True);
test!(Bit2, False, False, True, True);
test!(Bit2, False, True, False, False);
test!(Bit2, False, True, False, True);
test!(Bit2, False, True, True, True);
test!(Bit2, True, False, False, False);
test!(Bit2, True, False, False, True);
test!(Bit2, True, False, True, True);
test!(Bit2, True, True, False, False);
test!(Bit2, True, True, False, True);
test!(Bit2, True, True, True, True);
test!(Bit3, False, False, False, False);
test!(Bit3, False, False, False, True);
test!(Bit3, False, False, True, True);
test!(Bit3, False, True, False, False);
test!(Bit3, False, True, False, True);
test!(Bit3, False, True, True, True);
test!(Bit3, True, False, False, False);
test!(Bit3, True, False, False, True);
test!(Bit3, True, False, True, True);
test!(Bit3, True, True, False, False);
test!(Bit3, True, True, False, True);
test!(Bit3, True, True, True, True);
test!(Bit4, False, False, False, False);
test!(Bit4, False, False, False, True);
test!(Bit4, False, False, True, True);
test!(Bit4, False, True, False, False);
test!(Bit4, False, True, False, True);
test!(Bit4, False, True, True, True);
test!(Bit4, True, False, False, False);
test!(Bit4, True, False, False, True);
test!(Bit4, True, False, True, True);
test!(Bit4, True, True, False, False);
test!(Bit4, True, True, False, True);
test!(Bit4, True, True, True, True);
test!(Bit5, False, False, False, False);
test!(Bit5, False, False, False, True);
test!(Bit5, False, False, True, True);
test!(Bit5, False, True, False, False);
test!(Bit5, False, True, False, True);
test!(Bit5, False, True, True, True);
test!(Bit5, True, False, False, False);
test!(Bit5, True, False, False, True);
test!(Bit5, True, False, True, True);
test!(Bit5, True, True, False, False);
test!(Bit5, True, True, False, True);
test!(Bit5, True, True, True, True);
test!(Bit6, False, False, False, False);
test!(Bit6, False, False, False, True);
test!(Bit6, False, False, True, True);
test!(Bit6, False, True, False, False);
test!(Bit6, False, True, False, True);
test!(Bit6, False, True, True, True);
test!(Bit6, True, False, False, False);
test!(Bit6, True, False, False, True);
test!(Bit6, True, False, True, True);
test!(Bit6, True, True, False, False);
test!(Bit6, True, True, False, True);
test!(Bit6, True, True, True, True);
assert_eq!(count, 1);
});
2 changes: 0 additions & 2 deletions lib/fuzz/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
extern crate data_encoding;

use data_encoding::{DecodePartial, Encoding, Specification};

pub fn generate_encoding(data: &mut &[u8]) -> Encoding {
Expand Down
11 changes: 9 additions & 2 deletions lib/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ macro_rules! check {
};
}

#[cfg(feature = "v3-preview")]
pub mod v3_preview;

trait Static<T: Copy>: Copy {
fn val(self) -> T;
}
Expand Down Expand Up @@ -255,12 +258,12 @@ macro_rules! dispatch {
($body: expr) => { $body };
}

unsafe fn chunk_unchecked(x: &[u8], n: usize, i: usize) -> &[u8] {
unsafe fn chunk_unchecked<T>(x: &[T], n: usize, i: usize) -> &[T] {
debug_assert!((i + 1) * n <= x.len());
unsafe { core::slice::from_raw_parts(x.as_ptr().add(n * i), n) }
}

unsafe fn chunk_mut_unchecked(x: &mut [u8], n: usize, i: usize) -> &mut [u8] {
unsafe fn chunk_mut_unchecked<T>(x: &mut [T], n: usize, i: usize) -> &mut [T] {
debug_assert!((i + 1) * n <= x.len());
unsafe { core::slice::from_raw_parts_mut(x.as_mut_ptr().add(n * i), n) }
}
Expand All @@ -273,6 +276,7 @@ fn floor(x: usize, m: usize) -> usize {
x / m * m
}

#[inline]
fn vectorize<F: FnMut(usize)>(n: usize, bs: usize, mut f: F) {
for k in 0 .. n / bs {
for i in k * bs .. (k + 1) * bs {
Expand Down Expand Up @@ -362,6 +366,7 @@ fn order(msb: bool, n: usize, i: usize) -> usize {
}
}

#[inline]
fn enc(bit: usize) -> usize {
match bit {
1 | 2 | 4 => 1,
Expand All @@ -371,6 +376,7 @@ fn enc(bit: usize) -> usize {
}
}

#[inline]
fn dec(bit: usize) -> usize {
enc(bit) * 8 / bit
}
Expand Down Expand Up @@ -880,6 +886,7 @@ pub type InternalEncoding = &'static [u8];
// - width % dec(bit) == 0
// - for all x in separator values[x] is IGNORE
#[derive(Debug, Clone, PartialEq, Eq)]
#[repr(transparent)]
pub struct Encoding(#[doc(hidden)] pub InternalEncoding);

/// How to translate characters when decoding
Expand Down
Loading

0 comments on commit 2591b46

Please sign in to comment.