From f826805a560543187746e3460cfef6c963d6fa1a Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 20 Apr 2018 22:19:19 -0700 Subject: [PATCH 01/23] WIP work on storage adapter -- not working yet --- .gitignore | 1 + Cargo.lock | 1517 ++++++++++++++++++++++++++++++ Cargo.toml | 3 +- sync15-adapter/Cargo.toml | 18 + sync15-adapter/src/bso_record.rs | 115 +++ sync15-adapter/src/error.rs | 24 + sync15-adapter/src/key_bundle.rs | 136 +++ sync15-adapter/src/lib.rs | 30 + 8 files changed, 1843 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock create mode 100644 sync15-adapter/Cargo.toml create mode 100644 sync15-adapter/src/bso_record.rs create mode 100644 sync15-adapter/src/error.rs create mode 100644 sync15-adapter/src/key_bundle.rs create mode 100644 sync15-adapter/src/lib.rs diff --git a/.gitignore b/.gitignore index c328b56976..60b144290c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ website/build +target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..8de92fca4d --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1517 @@ +[[package]] +name = "adler32" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "arrayref" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base64" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base64" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "block-buffer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "boxlocker" +version = "0.1.0" +dependencies = [ + "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hawk 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", + "prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "build_const" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byte-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bytes" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cc" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "constant_time_eq" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "core-foundation" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "core-foundation-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crc" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-deque" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-deque" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crypto-mac" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "csv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "digest" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dtoa" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "encode_unicode" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "encoding_rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "error-chain" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure_derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gcc" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "generic-array" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hawk" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hkdf" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hmac" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crypto-mac 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "httparse" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hyper" +version = "0.11.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-tls" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "itoa" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazycell" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libflate" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crc 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memoffset" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime_guess" +version = "2.0.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "native-tls" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", + "schannel 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num-traits" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num_cpus" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl" +version = "0.9.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-sys" +version = "0.9.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "phf" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_codegen" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_generator" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_shared" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pkg-config" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "prettytable-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon-core" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "relay" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "reqwest" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libflate 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 2.0.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ring" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rust-crypto" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "safemem" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "schannel" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scoped-tls" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scopeguard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "security-framework" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "security-framework-sys" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive_internals" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_urlencoded" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "simple-error" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "siphasher" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "slab" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "slab" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sync15-adapter" +version = "0.1.0" +dependencies = [ + "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hkdf 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synstructure" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "take" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "term" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-executor" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-io" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-proto" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-timer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-tls" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-udp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "typenum" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicase" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicase" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "untrusted" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "url" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uuid" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "vcpkg" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "version_check" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6cbd0b9af8587c72beadc9f72d35b9fbb070982c9e6203e46e93f10df25f8f45" +"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" +"checksum backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbe525f66f42d207968308ee86bc2dd60aa5fab535b22e616323a173d097d8e" +"checksum backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "44585761d6161b0f57afc49482ab6bd067e4edef48c12a152c237eb0203f7661" +"checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" +"checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" +"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" +"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" +"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" +"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" +"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" +"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" +"checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" +"checksum cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8b9d2900f78631a5876dc5d6c9033ede027253efcd33dd36b1309fc6cab97ee0" +"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" +"checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" +"checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" +"checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" +"checksum crc 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd5d02c0aac6bd68393ed69e00bbc2457f3e89075c6349db7189618dc4ddc1d7" +"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" +"checksum crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c1bdc73742c36f7f35ebcda81dbb33a7e0d33757d03a06d9ddca762712ec5ea2" +"checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" +"checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81" +"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" +"checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b" +"checksum crypto-mac 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0999b4ff4d3446d4ddb19a63e9e00c1876e75cd7000d20e57a693b4b3f08d958" +"checksum csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7ef22b37c7a51c564a365892c012dc0271221fdcc64c69b19ba4d6fa8bd96d9c" +"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" +"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" +"checksum encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c088ec0ed2282dcd054f2c124c0327f953563e6c75fdc6ff5141779596289830" +"checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d" +"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" +"checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" +"checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" +"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c" +"checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +"checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" +"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +"checksum hawk 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae016790aafd7a6162922940898cd5eadc27e7622e992c899bc8e05ced6115ef" +"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +"checksum hkdf 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e55564e3d92b41ed9d41d9aaf147483dd5dbec2648a5d9af5e58405c2bc3a6c" +"checksum hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44f3bdb08579d99d7dc761c0e266f13b5f2ab8c8c703b9fc9ef333cd8f48f55e" +"checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37" +"checksum hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)" = "549dbb86397490ce69d908425b9beebc85bbaad25157d67479d4995bb56fdf9a" +"checksum hyper-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a5aa51f6ae9842239b0fac14af5f22123b8432b4cc774a44ff059fcba0f675ca" +"checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" +"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" +"checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" +"checksum itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c069bbec61e1ca5a596166e55dfe4773ff745c3d16b700013bcaff9a6df2c682" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" +"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" +"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" +"checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" +"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b" +"checksum libflate 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "1a429b86418868c7ea91ee50e9170683f47fd9d94f5375438ec86ec3adb74e8e" +"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" +"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +"checksum mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e00e17be181010a91dbfefb01660b17311059dc8c7f48b9017677721e732bd" +"checksum mime_guess 2.0.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "130ea3c9c1b65dba905ab5a4d9ac59234a9585c24d135f264e187fe7336febbd" +"checksum mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "6d771e3ef92d58a8da8df7d6976bfca9371ed1de6619d9d5a5ce5b1f29b85bfe" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f74dbadc8b43df7864539cedb7bc91345e532fdd913cfdc23ad94f4d2d40fbc0" +"checksum net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9044faf1413a1057267be51b5afba8eb1090bd2231c693664aa1db716fe1eae0" +"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dee092fcdf725aee04dd7da1d21debff559237d49ef1cb3e69bcb8ece44c7364" +"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +"checksum openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "a3605c298474a3aa69de92d21139fb5e2a81688d308262359d85cdd0d12a7985" +"checksum openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)" = "0bbd90640b148b46305c1691eed6039b5c8509bed16991e3562a01eeb76902a3" +"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cb325642290f28ee14d8c6201159949a872f220c62af6e110a56ea914fbe42fc" +"checksum phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d62594c0bb54c464f633175d502038177e90309daf2e0158be42ed5f023ce88f" +"checksum phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "6b07ffcc532ccc85e3afc45865469bf5d9e4ef5bfcf9622e3cfe80c2d275ec03" +"checksum phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "07e24b0ca9643bdecd0632f2b3da6b1b89bbb0030e0b992afc1113b23a7bc2f2" +"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" +"checksum prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "34dc1f4f6dddab3bf008ecfd4fd2a631b585fbf0af123f34c1324f51a034ff5f" +"checksum proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "49b6a521dc81b643e9a51e0d1cf05df46d5a2f3c0280ea72bcb68276ba64a118" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" +"checksum quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7b0ff51282f28dc1b53fd154298feaa2e77c5ea0dba68e1fd8b03b72fbe13d2a" +"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" +"checksum rayon 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a77c51c07654ddd93f6cb543c7a849863b03abc7e82591afda6dc8ad4ac3ac4a" +"checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8" +"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "241faa9a8ca28a03cbbb9815a5d085f271d4c0168a19181f106aa93240c22ddb" +"checksum ring 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a88f8432c04ba631e2434705bd848de0adf5b6e1a85a40671f5838e125e9d313" +"checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" +"checksum rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11fb43a206a04116ffd7cfcf9bcb941f8eb6cc7ff667272246b0a1c74259a3cb" +"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" +"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" +"checksum schannel 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "85fd9df495640643ad2d00443b3d78aae69802ad488debab4f1dd52fc1806ade" +"checksum scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8674d439c964889e2476f474a3bf198cc9e199e77499960893bac5de7e9218a4" +"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +"checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" +"checksum security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5421621e836278a0b139268f36eee0dc7e389b784dc3f79d8f11aabadf41bead" +"checksum serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)" = "5f751e9d57bd42502e4362b2d84f916ed9578e9a1a46852dcdeb6f91f6de7c14" +"checksum serde_derive 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)" = "81ec46cb12594da6750ad5a913f8175798c371d6c5ffd07f082ac4fea32789c3" +"checksum serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d30c4596450fd7bbda79ef15559683f9a79ac0193ea819db90000d7e1cae794" +"checksum serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7bf1cbb1387028a13739cb018ee0d9b3db534f22ca3c84a5904f7eadfde14e75" +"checksum serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce0fd303af908732989354c6f02e05e2e6d597152870f2c6990efb0577137480" +"checksum sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7daca11f2fdb8559c4f6c588386bed5e2ad4b6605c1442935a7f08144a918688" +"checksum simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "7779d1977a9e1e50bebb430a57114acc64bc4c40d6d8efb3e57893531d5fd895" +"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" +"checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" +"checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" +"checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +"checksum syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "91b52877572087400e83d24b9178488541e3d535259e04ff17a63df1e5ceff59" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +"checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" +"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" +"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" +"checksum tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "be15ef40f675c9fe66e354d74c73f3ed012ca1aa14d65846a33ee48f1ae8d922" +"checksum tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "aeeffbbb94209023feaef3c196a41cbcdafa06b4a6f893f68779bb5e53796f71" +"checksum tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8cac2a7883ff3567e9d66bb09100d09b33d90311feca0206c7ca034bc0c55113" +"checksum tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6af9eb326f64b2d6b68438e1953341e00ab3cf54de7e35d92bfc73af8555313a" +"checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" +"checksum tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3cedc8e5af5131dc3423ffa4f877cce78ad25259a9a62de0613735a13ebc64b" +"checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +"checksum tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec9b094851aadd2caf83ba3ad8e8c4ce65a42104f7b94d9e6550023f0407853f" +"checksum tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3d05cdd6a78005e535d2b27c21521bdf91fbb321027a62d8e178929d18966d" +"checksum tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "29a89e4ad0c8f1e4c9860e605c38c69bfdad3cccd4ea446e58ff588c1c07a397" +"checksum tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "772f4b04e560117fe3b0a53e490c16ddc8ba6ec437015d91fa385564996ed913" +"checksum tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "137bda266504893ac4774e0ec4c2108f7ccdbcb7ac8dced6305fe9e4e0b5041a" +"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" +"checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +"checksum unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284b6d3db520d67fbe88fd778c21510d1b0ba4a551e5d0fbb023d33405f6de8a" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" +"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" +"checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" +"checksum uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" +"checksum vcpkg 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7ed0f6789c8a85ca41bbc1c9d175422116a9869bd1cf31bb08e1493ecce60380" +"checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/Cargo.toml b/Cargo.toml index 1d94a30c98..361b6a7815 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] members = [ - "boxlocker" + "boxlocker", + "sync15-adapter" ] diff --git a/sync15-adapter/Cargo.toml b/sync15-adapter/Cargo.toml new file mode 100644 index 0000000000..75344aa710 --- /dev/null +++ b/sync15-adapter/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sync15-adapter" +version = "0.1.0" +authors = ["Thom Chiovoloni "] + +[dependencies] +base64 = "0.9.0" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +url = "1.6.0" +reqwest = "0.8.2" +hex = "0.3.1" +hkdf = "0.4" +sha2 = "0.7.0" +error_chain = "0.11" +openssl = "0.10" + diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs new file mode 100644 index 0000000000..7406ecb76e --- /dev/null +++ b/sync15-adapter/src/bso_record.rs @@ -0,0 +1,115 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use serde::de::{self, Deserialize, Deserializer, Visitor, MapAccess}; +use serde::ser::{Serialize, Serializer, SerializeStruct}; + +use serde_json; +use std::marker::PhantomData; +use std::ord::Ord; + +use error; +use key_bundle::KeyBundle; + +#[derive(Debug, Clone, Deserialize)] +pub struct BsoRecord where T: Serialize + Deserialize { + pub id: String, + + pub collection: Option, + pub modified: f64, + pub sortindex: Option, + pub ttl: Option, + + // We do some serde magic here with serde to parse the payload from JSON as we deserialize. + // This avoids having a separate intermediate type that only exists so that we can deserialize + // it's payload field as JSON (Especially since this one is going to exist more-or-less just so + // that we can decrypt the data... + #[serde(deserialize_with = "deserialize_json")] + pub payload: T, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct EncryptedPayload { + #[serde(rename = "IV")] + pub iv: String, + pub ciphertext: String, + pub hmac: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct MetaGlobalEngine { + pub version: usize, + #[serde(rename = "syncID")] + pub sync_id: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct MetaGlobalPayload { + #[serde(rename = "syncID")] + pub sync_id: String, + #[serde(rename = "storageVersion")] + pub storage_version: usize, + pub declined: Vec, + pub engines: HashMap, +} + +pub type EncryptedRecord = BsoRecord; +pub type MetaGlobalRecord = BsoRecord; + +// Custom deserializer to handle auto-deserializing the payload from JSON. +fn deserialize_json<'de, T, D>(deserializer: D) -> Result where T: Deserialize<'de>, D: Deserializer<'de> { + struct DeserializeNestedJson(PhantomData T>); + + impl<'de, T> Visitor<'de> for DeserializeNestedJson where T: Deserialize<'de> { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("The JSON-encoded payload as string") + } + + fn visit_str(self, value: &str) -> Result where E: de::Error { + serde_json::from_str(value) + } + } + + let visitor = DeserializeNestedJson(PhantomData); + deserializer.deserialize_str(visitor) +} + +// Custom serializer to handle auto-serializing the payload to JSON +impl Serialize for BsoRecord where T: Serialize { + fn serialize(&self, serializer: S) -> Result where S: Serializer { + // Serialize the object we hold in our payload to a string right away + let payload_json = serde_json::to_string(self.payload)?; + + // We always serialize id and payload, and serialize collection, ttl, and sortindex iff. + // they are present. Annoyingly, serialize_struct requires us tell how many we'll serialize + // up-front. + let num_fields = 2 + (self.collection.is_some() as usize) + + (self.ttl.is_some() as usize) + + (self.sortindex.is_some() as usize); + + // Note: The name here doesn't show up in the output. At least, not for JSON. + let mut state = serializer.serialize_struct("BsoRecord", num_fields)?; + state.serialize_field("id", &self.id)?; + state.serialize_field("payload", payload)?; + + if let &Some(ref collection) = &self.collection { + state.serialize_field("collection", collection)?; + } + if let &Some(ref sortindex) = &self.sortindex { + state.serialize_field("sortindex", sortindex)?; + } + if let &Some(ref ttl) = &self.ttl { + state.serialize_field("ttl", ttl)?; + } + state.end() + } +} + + + + + + diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs new file mode 100644 index 0000000000..1167bdb7c1 --- /dev/null +++ b/sync15-adapter/src/error.rs @@ -0,0 +1,24 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use error_chain; + +use base64; +use openssl; + +error_chain! { + foreign_links { + Base64Decode(::base64::DecodeError) + OpensslError(::openssl::error::ErrorStack) + BadCleartextUtf8(::std::string::FromUtf8Error) + } + errors { + BadKeyLength(which_key: &'static str, length: usize) { + description("Incorrect key length") + display("Incorrect key length for key {}: {}", which_key, length) + } + } +} + + diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs new file mode 100644 index 0000000000..b5e64f1ee8 --- /dev/null +++ b/sync15-adapter/src/key_bundle.rs @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use error::{Error, Result}; +use base64; +use openssl::{self, symm}; +use openssl::hash::MessageDigest; +use openssl::pkey::PKey; +use openssl::sign::Signer; + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct KeyBundle { + enc_key: Vec, + mac_key: Vec, +} + +impl KeyBundle { + + /// Construct a key bundle from the already-decoded encrypt and hmac keys. + /// Panics (asserts) if they aren't both 32 bytes. + pub fn new(enc: Vec, mac: Vec) -> Result { + if enc.len() != 32 { + // We probably should say which is bad... + return Err(Error::BadKeyLength(enc.len())); + } + if mac.len() != 32 { + return Err(Error::BadKeyLength(mac.len())); + } + Ok(KeyBundle { enc_key: enc, mac_key: mac }) + } + + pub fn new_random() -> Result { + let mut buffer = [0u8; 64]; + openssl::rand::rand_bytes(&mut buffer)?; + KeyBundle::from_ksync_bytes(&buffer) + } + + pub fn from_ksync_bytes(ksync: &[u8]) -> Result { + if ksync.len() != 64 { + return Err(Error::BadKeyLength(ksync.len())); + } + Ok(KeyBundle { + enc_key: ksync[0..32].into(), + mac_key: ksync[32..64].into() + }) + } + + pub fn from_ksync_base64(ksync: &str) -> Result { + let bytes = base64::decode_config(&ksync, base64::URL_SAFE_NO_PAD)?; + KeyBundle::from_ksync_bytes(&bytes) + } + + pub fn from_base64(enc: &str, mac: &str) -> Result { + let enc_bytes = base64::decode_config(&enc, base64::URL_SAFE_NO_PAD)?; + let mac_bytes = base64::decode_config(&mac, base64::URL_SAFE_NO_PAD)?; + KeyBundle::new(enc_bytes.into(), mac_bytes.into()); + } + + #[inline] + pub fn encryption_key(&self) -> &[u8] { + &self.enc_key + } + + #[inline] + pub fn hmac_key(&self) -> &[u8] { + &self.mac_key + } + + fn hmac(&self, ciphertext: &[u8], output: &mut [u8]): Result<()> { + let key = PKey::hmac(self.hmac_key())?; + let mut signer = Signer::new(MessageDigest::sha256(), &key)?; + signer.update(ciphertext)?; + let size = signer.sign(&output)?; + // This isn't an Err since it *really* shouldn't happen. 32 * 8 == 256, + // and so SHA256 should always output 256 bits of information. + assert!(size == 32, "sha256 digest is somehow not 32 bytes"); + Ok(()) + } + + pub fn hmac_to_vec(&self, ciphertext: &[u8]) -> Result> { + let mut out = vec![0u8; 32]; + let size = self.hmac(ciphertext, &out); + Ok(out) + } + + pub fn verify_hmac(&self, hmac: &[u8], ciphertext: &[u8]) -> Result { + let mut computed_hmac = [0u8; 32]; + self.hmac(ciphertext, &mut computed_hmac)?; + // The rust-openssl docs are pretty explicit that we shouldn't + // verify HMACs with ==, and should use openssl::memcmp::eq. + // This is presumably related to sidechannels? I don't think we + // actually need to be concerned about them, but who knows. + openssl::memcmp::eq(&hmac, &computed_hmac) + } + + /// Decrypt the provided ciphertext with the given iv, and decodes the + /// result as a utf8 string. Important: Caller must check verify_hmac first! + pub fn decrypt(&self, ciphertext: &str, iv: &[u8]) -> Result { + let cleartext_bytes = symm::decrypt(symm::Cipher::aes_256_cbc(), + self.encryption_key(), + Some(iv), + ciphertext)?; + let cleartext = String::from_utf8(cleartext_bytes)?; + Ok(cleartext) + } + + /// Encrypt using the provided IV. + pub fn encrypt_with_iv(&self, cleartext_bytes: &[u8], iv: &[u8]) -> Result> { + let cleartext = symm::encypt(symm::Cipher::aes_256_cbc(), + self.encryption_key(), + Some(iv), + cleartext)?; + Ok(cleartext) + } + + /// Generate a random iv and encrypt with it. Return both the encrypted bytes + /// and the generated iv. + pub fn encrypt_rand_iv(&self, cleartext: &[u8]) -> Result<(Vec, [u8; 16])> { + let mut iv = [0u8; 16]; + openssl::rand::rand_bytes(&mut iv)?; + let ciphertext = symm::encypt(symm::Cipher::aes_256_cbc(), + self.encryption_key(), + Some(&iv), + cleartext)?; + Ok((ciphertext, iv)) + } + + + + + + +} + + diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs new file mode 100644 index 0000000000..f61b2409ab --- /dev/null +++ b/sync15-adapter/src/lib.rs @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +extern crate serde; +extern crate base64; +extern crate openssl; + +#[macro_use] +extern crate serde_derive; + + +#[macro_use] +extern crate serde_json; + +pub mod key_bundle; +pub mod error; +pub mod record; + + + + + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} From d75ed6c6db4420bec37db08b255652690d7433e4 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 23 Apr 2018 12:50:45 -0700 Subject: [PATCH 02/23] Fix compiler errors and add serialization tests --- Cargo.lock | 139 ++++++++++--------------------- sync15-adapter/Cargo.toml | 2 +- sync15-adapter/src/bso_record.rs | 74 ++++++++++++---- sync15-adapter/src/error.rs | 6 +- sync15-adapter/src/key_bundle.rs | 72 +++++++--------- sync15-adapter/src/lib.rs | 5 +- 6 files changed, 143 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8de92fca4d..d1cf2cee16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ dependencies = [ [[package]] name = "atty" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "base64" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -88,16 +88,16 @@ dependencies = [ name = "boxlocker" version = "0.1.0" dependencies = [ - "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "hawk 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", "prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", "simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -280,22 +280,11 @@ dependencies = [ ] [[package]] -name = "failure" -version = "0.1.1" +name = "error-chain" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "failure_derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -404,7 +393,7 @@ name = "hyper" version = "0.11.25" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -621,11 +610,6 @@ name = "nodrop" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "num-traits" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "num_cpus" version = "1.8.0" @@ -646,6 +630,18 @@ dependencies = [ "openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "openssl" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "openssl-sys" version = "0.9.28" @@ -707,7 +703,7 @@ name = "prettytable-rs" version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -725,12 +721,7 @@ dependencies = [ [[package]] name = "quote" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "quote" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -819,8 +810,8 @@ dependencies = [ "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", "serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -909,16 +900,16 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.41" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.41" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -934,13 +925,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -950,7 +940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -990,23 +980,13 @@ name = "smallvec" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "syn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "syn" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1014,35 +994,19 @@ dependencies = [ name = "sync15-adapter" version = "0.1.0" dependencies = [ - "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hkdf 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "synstructure" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "take" version = "0.1.0" @@ -1274,11 +1238,6 @@ name = "unicode-width" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "unicode-xid" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "unicode-xid" version = "0.1.0" @@ -1359,11 +1318,11 @@ dependencies = [ "checksum adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6cbd0b9af8587c72beadc9f72d35b9fbb070982c9e6203e46e93f10df25f8f45" "checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" -"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" +"checksum atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6609a866dd1a1b2d0ee1362195bf3e4f6438abb2d80120b83b1e1f4fb6476dd0" "checksum backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbe525f66f42d207968308ee86bc2dd60aa5fab535b22e616323a173d097d8e" "checksum backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "44585761d6161b0f57afc49482ab6bd067e4edef48c12a152c237eb0203f7661" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" -"checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" +"checksum base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9263aa6a38da271eec5c91a83ce1e800f093c8535788d403d626d8d5c3f8f007" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" "checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" @@ -1390,8 +1349,7 @@ dependencies = [ "checksum encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c088ec0ed2282dcd054f2c124c0327f953563e6c75fdc6ff5141779596289830" "checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d" "checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" -"checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" -"checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" +"checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" @@ -1431,8 +1389,8 @@ dependencies = [ "checksum native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f74dbadc8b43df7864539cedb7bc91345e532fdd913cfdc23ad94f4d2d40fbc0" "checksum net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9044faf1413a1057267be51b5afba8eb1090bd2231c693664aa1db716fe1eae0" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" -"checksum num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dee092fcdf725aee04dd7da1d21debff559237d49ef1cb3e69bcb8ece44c7364" "checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +"checksum openssl 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "63246f69962e8d5ef865f82a65241d6483c8a2905a1801e2f7feb5d187d51320" "checksum openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "a3605c298474a3aa69de92d21139fb5e2a81688d308262359d85cdd0d12a7985" "checksum openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)" = "0bbd90640b148b46305c1691eed6039b5c8509bed16991e3562a01eeb76902a3" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" @@ -1443,8 +1401,7 @@ dependencies = [ "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "34dc1f4f6dddab3bf008ecfd4fd2a631b585fbf0af123f34c1324f51a034ff5f" "checksum proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "49b6a521dc81b643e9a51e0d1cf05df46d5a2f3c0280ea72bcb68276ba64a118" -"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" -"checksum quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7b0ff51282f28dc1b53fd154298feaa2e77c5ea0dba68e1fd8b03b72fbe13d2a" +"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" "checksum rayon 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a77c51c07654ddd93f6cb543c7a849863b03abc7e82591afda6dc8ad4ac3ac4a" @@ -1464,10 +1421,10 @@ dependencies = [ "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" "checksum security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5421621e836278a0b139268f36eee0dc7e389b784dc3f79d8f11aabadf41bead" -"checksum serde 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)" = "5f751e9d57bd42502e4362b2d84f916ed9578e9a1a46852dcdeb6f91f6de7c14" -"checksum serde_derive 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)" = "81ec46cb12594da6750ad5a913f8175798c371d6c5ffd07f082ac4fea32789c3" +"checksum serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "0c855d888276f20d140223bd06515e5bf1647fd6d02593cb5792466d9a8ec2d0" +"checksum serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "aa113e5fc4b008a626ba2bbd41330b56c9987d667f79f7b243e5a2d03d91ed1c" "checksum serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d30c4596450fd7bbda79ef15559683f9a79ac0193ea819db90000d7e1cae794" -"checksum serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7bf1cbb1387028a13739cb018ee0d9b3db534f22ca3c84a5904f7eadfde14e75" +"checksum serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8c6c4e049dc657a99e394bd85c22acbf97356feeec6dbf44150f2dcf79fb3118" "checksum serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce0fd303af908732989354c6f02e05e2e6d597152870f2c6990efb0577137480" "checksum sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7daca11f2fdb8559c4f6c588386bed5e2ad4b6605c1442935a7f08144a918688" "checksum simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "7779d1977a9e1e50bebb430a57114acc64bc4c40d6d8efb3e57893531d5fd895" @@ -1475,10 +1432,7 @@ dependencies = [ "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" "checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" -"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "91b52877572087400e83d24b9178488541e3d535259e04ff17a63df1e5ceff59" -"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -"checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" "checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" @@ -1502,7 +1456,6 @@ dependencies = [ "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" -"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" "checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" diff --git a/sync15-adapter/Cargo.toml b/sync15-adapter/Cargo.toml index 75344aa710..9b3ad1cba3 100644 --- a/sync15-adapter/Cargo.toml +++ b/sync15-adapter/Cargo.toml @@ -13,6 +13,6 @@ reqwest = "0.8.2" hex = "0.3.1" hkdf = "0.4" sha2 = "0.7.0" -error_chain = "0.11" +error-chain = "0.11" openssl = "0.10" diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 7406ecb76e..83f84bc63a 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -2,18 +2,23 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use serde::de::{self, Deserialize, Deserializer, Visitor, MapAccess}; -use serde::ser::{Serialize, Serializer, SerializeStruct}; +use serde::de::{self, Deserialize, Deserializer, Visitor}; +use serde::ser::{self, Serialize, Serializer, SerializeStruct}; +// use serde_derive; use serde_json; + +use std::collections::HashMap; + use std::marker::PhantomData; -use std::ord::Ord; +// use std::cmp::Ord; +use std::fmt; -use error; -use key_bundle::KeyBundle; +// use error; +// use key_bundle::KeyBundle; #[derive(Debug, Clone, Deserialize)] -pub struct BsoRecord where T: Serialize + Deserialize { +pub struct BsoRecord where for<'a> T: Deserialize<'a> { pub id: String, pub collection: Option, @@ -33,8 +38,8 @@ pub struct BsoRecord where T: Serialize + Deserialize { pub struct EncryptedPayload { #[serde(rename = "IV")] pub iv: String, - pub ciphertext: String, pub hmac: String, + pub ciphertext: String, } #[derive(Deserialize, Serialize, Clone, Debug)] @@ -58,10 +63,10 @@ pub type EncryptedRecord = BsoRecord; pub type MetaGlobalRecord = BsoRecord; // Custom deserializer to handle auto-deserializing the payload from JSON. -fn deserialize_json<'de, T, D>(deserializer: D) -> Result where T: Deserialize<'de>, D: Deserializer<'de> { +fn deserialize_json<'de, T, D>(deserializer: D) -> Result where for <'a> T: Deserialize<'a>, D: Deserializer<'de> { struct DeserializeNestedJson(PhantomData T>); - impl<'de, T> Visitor<'de> for DeserializeNestedJson where T: Deserialize<'de> { + impl<'de, T> Visitor<'de> for DeserializeNestedJson where for<'a> T: Deserialize<'a> { type Value = T; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { @@ -69,7 +74,7 @@ fn deserialize_json<'de, T, D>(deserializer: D) -> Result where T: } fn visit_str(self, value: &str) -> Result where E: de::Error { - serde_json::from_str(value) + serde_json::from_str(&value).map_err(|e| de::Error::custom(e)) } } @@ -78,10 +83,10 @@ fn deserialize_json<'de, T, D>(deserializer: D) -> Result where T: } // Custom serializer to handle auto-serializing the payload to JSON -impl Serialize for BsoRecord where T: Serialize { +impl Serialize for BsoRecord where T: Serialize, for<'a> T: Deserialize<'a> { fn serialize(&self, serializer: S) -> Result where S: Serializer { - // Serialize the object we hold in our payload to a string right away - let payload_json = serde_json::to_string(self.payload)?; + // Serialize the object we hold in our payload to a string right away. + let payload_json = serde_json::to_string(&self.payload).map_err(|e| ser::Error::custom(e))?; // We always serialize id and payload, and serialize collection, ttl, and sortindex iff. // they are present. Annoyingly, serialize_struct requires us tell how many we'll serialize @@ -93,7 +98,7 @@ impl Serialize for BsoRecord where T: Serialize { // Note: The name here doesn't show up in the output. At least, not for JSON. let mut state = serializer.serialize_struct("BsoRecord", num_fields)?; state.serialize_field("id", &self.id)?; - state.serialize_field("payload", payload)?; + state.serialize_field("payload", &payload_json)?; if let &Some(ref collection) = &self.collection { state.serialize_field("collection", collection)?; @@ -110,6 +115,41 @@ impl Serialize for BsoRecord where T: Serialize { - - - +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_deserialize_enc() { + let serialized = r#"{ + "id": "1234", + "collection": "passwords", + "modified": 12344321.0, + "payload": "{\"IV\": \"aaaaa\", \"hmac\": \"bbbbb\", \"ciphertext\": \"ccccc\"}" + }"#; + let record: EncryptedRecord = serde_json::from_str(serialized).unwrap(); + assert_eq!(&record.id, "1234"); + assert_eq!(&record.collection.unwrap(), "passwords"); + assert_eq!(record.modified, 12344321.0); + assert_eq!(&record.payload.iv, "aaaaa"); + assert_eq!(&record.payload.hmac, "bbbbb"); + assert_eq!(&record.payload.ciphertext, "ccccc"); + } + #[test] + fn test_serialize_enc() { + let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"hmac\":\"bbbbb\",\"ciphertext\":\"ccccc\"}","collection":"passwords"}"#; + let record = EncryptedRecord { + id: "1234".into(), + modified: 999.0, // shouldn't be serialized by client no matter what it's value is + collection: Some("passwords".into()), + sortindex: None, + ttl: None, + payload: EncryptedPayload { + iv: "aaaaa".into(), + hmac: "bbbbb".into(), + ciphertext: "ccccc".into(), + } + }; + let actual = serde_json::to_string(&record).unwrap(); + assert_eq!(actual, goal); + } +} diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index 1167bdb7c1..0e340564c4 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -9,9 +9,9 @@ use openssl; error_chain! { foreign_links { - Base64Decode(::base64::DecodeError) - OpensslError(::openssl::error::ErrorStack) - BadCleartextUtf8(::std::string::FromUtf8Error) + Base64Decode(::base64::DecodeError); + OpensslError(::openssl::error::ErrorStack); + BadCleartextUtf8(::std::string::FromUtf8Error); } errors { BadKeyLength(which_key: &'static str, length: usize) { diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index b5e64f1ee8..d2eb10b01d 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use error::{Error, Result}; +use error::{Error, Result, ErrorKind}; use base64; use openssl::{self, symm}; use openssl::hash::MessageDigest; @@ -22,10 +22,10 @@ impl KeyBundle { pub fn new(enc: Vec, mac: Vec) -> Result { if enc.len() != 32 { // We probably should say which is bad... - return Err(Error::BadKeyLength(enc.len())); + return Err(ErrorKind::BadKeyLength("enc_key", enc.len()).into()); } if mac.len() != 32 { - return Err(Error::BadKeyLength(mac.len())); + return Err(ErrorKind::BadKeyLength("mac_key", mac.len()).into()); } Ok(KeyBundle { enc_key: enc, mac_key: mac }) } @@ -38,7 +38,7 @@ impl KeyBundle { pub fn from_ksync_bytes(ksync: &[u8]) -> Result { if ksync.len() != 64 { - return Err(Error::BadKeyLength(ksync.len())); + return Err(ErrorKind::BadKeyLength("kSync", ksync.len()).into()); } Ok(KeyBundle { enc_key: ksync[0..32].into(), @@ -54,7 +54,7 @@ impl KeyBundle { pub fn from_base64(enc: &str, mac: &str) -> Result { let enc_bytes = base64::decode_config(&enc, base64::URL_SAFE_NO_PAD)?; let mac_bytes = base64::decode_config(&mac, base64::URL_SAFE_NO_PAD)?; - KeyBundle::new(enc_bytes.into(), mac_bytes.into()); + KeyBundle::new(enc_bytes.into(), mac_bytes.into()) } #[inline] @@ -67,36 +67,29 @@ impl KeyBundle { &self.mac_key } - fn hmac(&self, ciphertext: &[u8], output: &mut [u8]): Result<()> { + /// Returns the 32 byte digest by value since it's small enough to be passed + /// around cheaply, and easily convertable into a slice or vec if you want. + fn hmac(&self, ciphertext: &[u8]) -> Result<[u8; 32]> { + let mut out = [0u8; 32]; let key = PKey::hmac(self.hmac_key())?; let mut signer = Signer::new(MessageDigest::sha256(), &key)?; signer.update(ciphertext)?; - let size = signer.sign(&output)?; - // This isn't an Err since it *really* shouldn't happen. 32 * 8 == 256, - // and so SHA256 should always output 256 bits of information. - assert!(size == 32, "sha256 digest is somehow not 32 bytes"); - Ok(()) - } - - pub fn hmac_to_vec(&self, ciphertext: &[u8]) -> Result> { - let mut out = vec![0u8; 32]; - let size = self.hmac(ciphertext, &out); + let size = signer.sign(&mut out)?; + // This isn't an Err since it really should not be possible. + assert!(size == 32, "Somehow the 256 bits from sha256 do not add up into 32 bytes..."); Ok(out) } - pub fn verify_hmac(&self, hmac: &[u8], ciphertext: &[u8]) -> Result { - let mut computed_hmac = [0u8; 32]; - self.hmac(ciphertext, &mut computed_hmac)?; - // The rust-openssl docs are pretty explicit that we shouldn't - // verify HMACs with ==, and should use openssl::memcmp::eq. - // This is presumably related to sidechannels? I don't think we - // actually need to be concerned about them, but who knows. - openssl::memcmp::eq(&hmac, &computed_hmac) + pub fn verify_hmac(&self, expected_hmac: &[u8], ciphertext_base64: &str) -> Result { + let computed_hmac = self.hmac(ciphertext_base64.as_bytes())?; + // I suspect this is unnecessary for our case, but the rust-openssl docs + // want us to use this over == to avoid sidechannels, and who am I to argue? + Ok(openssl::memcmp::eq(&expected_hmac, &computed_hmac)) } /// Decrypt the provided ciphertext with the given iv, and decodes the /// result as a utf8 string. Important: Caller must check verify_hmac first! - pub fn decrypt(&self, ciphertext: &str, iv: &[u8]) -> Result { + pub fn decrypt(&self, ciphertext: &[u8], iv: &[u8]) -> Result { let cleartext_bytes = symm::decrypt(symm::Cipher::aes_256_cbc(), self.encryption_key(), Some(iv), @@ -106,31 +99,30 @@ impl KeyBundle { } /// Encrypt using the provided IV. - pub fn encrypt_with_iv(&self, cleartext_bytes: &[u8], iv: &[u8]) -> Result> { - let cleartext = symm::encypt(symm::Cipher::aes_256_cbc(), - self.encryption_key(), - Some(iv), - cleartext)?; - Ok(cleartext) + pub fn encrypt_bytes_with_iv(&self, cleartext_bytes: &[u8], iv: &[u8]) -> Result> { + let ciphertext = symm::encrypt(symm::Cipher::aes_256_cbc(), + self.encryption_key(), + Some(iv), + cleartext_bytes)?; + Ok(ciphertext) } /// Generate a random iv and encrypt with it. Return both the encrypted bytes /// and the generated iv. - pub fn encrypt_rand_iv(&self, cleartext: &[u8]) -> Result<(Vec, [u8; 16])> { + pub fn encrypt_bytes_rand_iv(&self, cleartext_bytes: &[u8]) -> Result<(Vec, [u8; 16])> { let mut iv = [0u8; 16]; openssl::rand::rand_bytes(&mut iv)?; - let ciphertext = symm::encypt(symm::Cipher::aes_256_cbc(), - self.encryption_key(), - Some(&iv), - cleartext)?; + let ciphertext = self.encrypt_bytes_with_iv(cleartext_bytes, &iv)?; Ok((ciphertext, iv)) } + pub fn encrypt_with_iv(&self, cleartext: &str, iv: &[u8]) -> Result> { + self.encrypt_bytes_with_iv(cleartext.as_bytes(), iv) + } - - - - + pub fn encrypt_rand_iv(&self, cleartext: &str) -> Result<(Vec, [u8; 16])> { + self.encrypt_bytes_rand_iv(cleartext.as_bytes()) + } } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index f61b2409ab..969b5befcc 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -13,9 +13,12 @@ extern crate serde_derive; #[macro_use] extern crate serde_json; +#[macro_use] +extern crate error_chain; + pub mod key_bundle; pub mod error; -pub mod record; +pub mod bso_record; From f9d543676f14cd005aa417c48665d35996a2f810 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 23 Apr 2018 12:52:44 -0700 Subject: [PATCH 03/23] Don't version Cargo.lock I'm not exactly sure if this is a good idea, really. --- .gitignore | 1 + Cargo.lock | 1470 ---------------------------------------------------- 2 files changed, 1 insertion(+), 1470 deletions(-) delete mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index 60b144290c..84640e8f2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ website/build target +Cargo.lock diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index d1cf2cee16..0000000000 --- a/Cargo.lock +++ /dev/null @@ -1,1470 +0,0 @@ -[[package]] -name = "adler32" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "arrayref" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "arrayvec" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "atty" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "base64" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "base64" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bitflags" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "bitflags" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "block-buffer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "boxlocker" -version = "0.1.0" -dependencies = [ - "base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hawk 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", - "prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", - "reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", - "simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "build_const" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "byte-tools" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "byteorder" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "bytes" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "cc" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "cfg-if" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "constant_time_eq" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "core-foundation" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "core-foundation-sys" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crc" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-deque" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-deque" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-utils" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-utils" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crypto-mac" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "csv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "digest" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "dtoa" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "encode_unicode" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "encoding_rs" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "error-chain" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "error-chain" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "futures" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "futures-cpupool" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "gcc" -version = "0.3.54" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "generic-array" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hawk" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", - "ring 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hex" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "hkdf" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hmac" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crypto-mac 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "httparse" -version = "1.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "hyper" -version = "0.11.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hyper-tls" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "idna" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "iovec" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "itoa" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "itoa" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "lazy_static" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "lazy_static" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "lazycell" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libc" -version = "0.2.40" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libflate" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crc 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "matches" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "memchr" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memoffset" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "mime" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime_guess" -version = "2.0.0-alpha.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mio" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "miow" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "native-tls" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", - "schannel 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "net2" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nodrop" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "num_cpus" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl" -version = "0.9.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-sys" -version = "0.9.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "vcpkg 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "phf" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_codegen" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_generator" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_shared" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "pkg-config" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "prettytable-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "proc-macro2" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "quote" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand" -version = "0.3.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rayon" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rayon-core" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "redox_syscall" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "redox_termios" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "relay" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "remove_dir_all" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "reqwest" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libflate 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 2.0.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ring" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rust-crypto" -version = "0.2.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "rustc-serialize" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "safemem" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "schannel" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "scoped-tls" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "scopeguard" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "security-framework" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "security-framework-sys" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "serde" -version = "1.0.43" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "serde_derive" -version = "1.0.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "serde_derive_internals" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "serde_json" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "serde_urlencoded" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sha2" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "simple-error" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "siphasher" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "slab" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "slab" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "smallvec" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "syn" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sync15-adapter" -version = "0.1.0" -dependencies = [ - "base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hkdf 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", - "reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "take" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "term" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termion" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "time" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-core" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-executor" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-io" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-proto" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-reactor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-service" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-tcp" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-threadpool" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-timer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-tls" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tokio-udp" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "typenum" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicase" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicase" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicode-normalization" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-width" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "untrusted" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "url" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "uuid" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "vcpkg" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "version_check" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[metadata] -"checksum adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6cbd0b9af8587c72beadc9f72d35b9fbb070982c9e6203e46e93f10df25f8f45" -"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" -"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" -"checksum atty 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6609a866dd1a1b2d0ee1362195bf3e4f6438abb2d80120b83b1e1f4fb6476dd0" -"checksum backtrace 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbe525f66f42d207968308ee86bc2dd60aa5fab535b22e616323a173d097d8e" -"checksum backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "44585761d6161b0f57afc49482ab6bd067e4edef48c12a152c237eb0203f7661" -"checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" -"checksum base64 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9263aa6a38da271eec5c91a83ce1e800f093c8535788d403d626d8d5c3f8f007" -"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" -"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" -"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" -"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" -"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" -"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" -"checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" -"checksum cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8b9d2900f78631a5876dc5d6c9033ede027253efcd33dd36b1309fc6cab97ee0" -"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" -"checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" -"checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" -"checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" -"checksum crc 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd5d02c0aac6bd68393ed69e00bbc2457f3e89075c6349db7189618dc4ddc1d7" -"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" -"checksum crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c1bdc73742c36f7f35ebcda81dbb33a7e0d33757d03a06d9ddca762712ec5ea2" -"checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" -"checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81" -"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" -"checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b" -"checksum crypto-mac 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0999b4ff4d3446d4ddb19a63e9e00c1876e75cd7000d20e57a693b4b3f08d958" -"checksum csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7ef22b37c7a51c564a365892c012dc0271221fdcc64c69b19ba4d6fa8bd96d9c" -"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" -"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" -"checksum encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c088ec0ed2282dcd054f2c124c0327f953563e6c75fdc6ff5141779596289830" -"checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d" -"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" -"checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" -"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" -"checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c" -"checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" -"checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" -"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" -"checksum hawk 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae016790aafd7a6162922940898cd5eadc27e7622e992c899bc8e05ced6115ef" -"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" -"checksum hkdf 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e55564e3d92b41ed9d41d9aaf147483dd5dbec2648a5d9af5e58405c2bc3a6c" -"checksum hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44f3bdb08579d99d7dc761c0e266f13b5f2ab8c8c703b9fc9ef333cd8f48f55e" -"checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37" -"checksum hyper 0.11.25 (registry+https://github.com/rust-lang/crates.io-index)" = "549dbb86397490ce69d908425b9beebc85bbaad25157d67479d4995bb56fdf9a" -"checksum hyper-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a5aa51f6ae9842239b0fac14af5f22123b8432b4cc774a44ff059fcba0f675ca" -"checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" -"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" -"checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" -"checksum itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c069bbec61e1ca5a596166e55dfe4773ff745c3d16b700013bcaff9a6df2c682" -"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" -"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" -"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" -"checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" -"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b" -"checksum libflate 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "1a429b86418868c7ea91ee50e9170683f47fd9d94f5375438ec86ec3adb74e8e" -"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" -"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" -"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" -"checksum mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e00e17be181010a91dbfefb01660b17311059dc8c7f48b9017677721e732bd" -"checksum mime_guess 2.0.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "130ea3c9c1b65dba905ab5a4d9ac59234a9585c24d135f264e187fe7336febbd" -"checksum mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "6d771e3ef92d58a8da8df7d6976bfca9371ed1de6619d9d5a5ce5b1f29b85bfe" -"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" -"checksum native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f74dbadc8b43df7864539cedb7bc91345e532fdd913cfdc23ad94f4d2d40fbc0" -"checksum net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9044faf1413a1057267be51b5afba8eb1090bd2231c693664aa1db716fe1eae0" -"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" -"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" -"checksum openssl 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "63246f69962e8d5ef865f82a65241d6483c8a2905a1801e2f7feb5d187d51320" -"checksum openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "a3605c298474a3aa69de92d21139fb5e2a81688d308262359d85cdd0d12a7985" -"checksum openssl-sys 0.9.28 (registry+https://github.com/rust-lang/crates.io-index)" = "0bbd90640b148b46305c1691eed6039b5c8509bed16991e3562a01eeb76902a3" -"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" -"checksum phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cb325642290f28ee14d8c6201159949a872f220c62af6e110a56ea914fbe42fc" -"checksum phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d62594c0bb54c464f633175d502038177e90309daf2e0158be42ed5f023ce88f" -"checksum phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "6b07ffcc532ccc85e3afc45865469bf5d9e4ef5bfcf9622e3cfe80c2d275ec03" -"checksum phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "07e24b0ca9643bdecd0632f2b3da6b1b89bbb0030e0b992afc1113b23a7bc2f2" -"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" -"checksum prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "34dc1f4f6dddab3bf008ecfd4fd2a631b585fbf0af123f34c1324f51a034ff5f" -"checksum proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "49b6a521dc81b643e9a51e0d1cf05df46d5a2f3c0280ea72bcb68276ba64a118" -"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" -"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" -"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" -"checksum rayon 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a77c51c07654ddd93f6cb543c7a849863b03abc7e82591afda6dc8ad4ac3ac4a" -"checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8" -"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" -"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" -"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" -"checksum reqwest 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "241faa9a8ca28a03cbbb9815a5d085f271d4c0168a19181f106aa93240c22ddb" -"checksum ring 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a88f8432c04ba631e2434705bd848de0adf5b6e1a85a40671f5838e125e9d313" -"checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" -"checksum rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11fb43a206a04116ffd7cfcf9bcb941f8eb6cc7ff667272246b0a1c74259a3cb" -"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" -"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" -"checksum schannel 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "85fd9df495640643ad2d00443b3d78aae69802ad488debab4f1dd52fc1806ade" -"checksum scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8674d439c964889e2476f474a3bf198cc9e199e77499960893bac5de7e9218a4" -"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" -"checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" -"checksum security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5421621e836278a0b139268f36eee0dc7e389b784dc3f79d8f11aabadf41bead" -"checksum serde 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "0c855d888276f20d140223bd06515e5bf1647fd6d02593cb5792466d9a8ec2d0" -"checksum serde_derive 1.0.43 (registry+https://github.com/rust-lang/crates.io-index)" = "aa113e5fc4b008a626ba2bbd41330b56c9987d667f79f7b243e5a2d03d91ed1c" -"checksum serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d30c4596450fd7bbda79ef15559683f9a79ac0193ea819db90000d7e1cae794" -"checksum serde_json 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8c6c4e049dc657a99e394bd85c22acbf97356feeec6dbf44150f2dcf79fb3118" -"checksum serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce0fd303af908732989354c6f02e05e2e6d597152870f2c6990efb0577137480" -"checksum sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7daca11f2fdb8559c4f6c588386bed5e2ad4b6605c1442935a7f08144a918688" -"checksum simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "7779d1977a9e1e50bebb430a57114acc64bc4c40d6d8efb3e57893531d5fd895" -"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" -"checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" -"checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" -"checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" -"checksum syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "91b52877572087400e83d24b9178488541e3d535259e04ff17a63df1e5ceff59" -"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" -"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -"checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" -"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" -"checksum tokio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "be15ef40f675c9fe66e354d74c73f3ed012ca1aa14d65846a33ee48f1ae8d922" -"checksum tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "aeeffbbb94209023feaef3c196a41cbcdafa06b4a6f893f68779bb5e53796f71" -"checksum tokio-executor 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8cac2a7883ff3567e9d66bb09100d09b33d90311feca0206c7ca034bc0c55113" -"checksum tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6af9eb326f64b2d6b68438e1953341e00ab3cf54de7e35d92bfc73af8555313a" -"checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" -"checksum tokio-reactor 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3cedc8e5af5131dc3423ffa4f877cce78ad25259a9a62de0613735a13ebc64b" -"checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" -"checksum tokio-tcp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ec9b094851aadd2caf83ba3ad8e8c4ce65a42104f7b94d9e6550023f0407853f" -"checksum tokio-threadpool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3d05cdd6a78005e535d2b27c21521bdf91fbb321027a62d8e178929d18966d" -"checksum tokio-timer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "29a89e4ad0c8f1e4c9860e605c38c69bfdad3cccd4ea446e58ff588c1c07a397" -"checksum tokio-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "772f4b04e560117fe3b0a53e490c16ddc8ba6ec437015d91fa385564996ed913" -"checksum tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "137bda266504893ac4774e0ec4c2108f7ccdbcb7ac8dced6305fe9e4e0b5041a" -"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" -"checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" -"checksum unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284b6d3db520d67fbe88fd778c21510d1b0ba4a551e5d0fbb023d33405f6de8a" -"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -"checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" -"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" -"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" -"checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" -"checksum uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" -"checksum vcpkg 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7ed0f6789c8a85ca41bbc1c9d175422116a9869bd1cf31bb08e1493ecce60380" -"checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d" -"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" -"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" From 6de7d6874728afcabb2deacfe2f98425bd8459b1 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 23 Apr 2018 12:54:52 -0700 Subject: [PATCH 04/23] Fix compiler warnings --- sync15-adapter/src/bso_record.rs | 7 ------- sync15-adapter/src/error.rs | 5 ----- sync15-adapter/src/key_bundle.rs | 2 +- sync15-adapter/src/lib.rs | 2 -- 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 83f84bc63a..b9ce1bfc9d 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -4,19 +4,12 @@ use serde::de::{self, Deserialize, Deserializer, Visitor}; use serde::ser::{self, Serialize, Serializer, SerializeStruct}; - -// use serde_derive; use serde_json; use std::collections::HashMap; - use std::marker::PhantomData; -// use std::cmp::Ord; use std::fmt; -// use error; -// use key_bundle::KeyBundle; - #[derive(Debug, Clone, Deserialize)] pub struct BsoRecord where for<'a> T: Deserialize<'a> { pub id: String, diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index 0e340564c4..8a1f8df59f 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -2,11 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use error_chain; - -use base64; -use openssl; - error_chain! { foreign_links { Base64Decode(::base64::DecodeError); diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index d2eb10b01d..68b418d0ae 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use error::{Error, Result, ErrorKind}; +use error::{Result, ErrorKind}; use base64; use openssl::{self, symm}; use openssl::hash::MessageDigest; diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 969b5befcc..6787f8a4be 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -9,8 +9,6 @@ extern crate openssl; #[macro_use] extern crate serde_derive; - -#[macro_use] extern crate serde_json; #[macro_use] From bafdddafc4c170b7aedbeaa5ff016bdd1118f385 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 23 Apr 2018 13:35:31 -0700 Subject: [PATCH 05/23] Keybundle tests --- sync15-adapter/src/bso_record.rs | 1 + sync15-adapter/src/key_bundle.rs | 87 +++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index b9ce1bfc9d..b872ab8948 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -129,6 +129,7 @@ mod tests { } #[test] fn test_serialize_enc() { + // This is sensitive to the order that the fields appear in in EncryptedPayload, unfortunately let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"hmac\":\"bbbbb\",\"ciphertext\":\"ccccc\"}","collection":"passwords"}"#; let record = EncryptedRecord { id: "1234".into(), diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index 68b418d0ae..1fc3b41f78 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -8,6 +8,7 @@ use openssl::{self, symm}; use openssl::hash::MessageDigest; use openssl::pkey::PKey; use openssl::sign::Signer; +use std::fmt::Write; #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub struct KeyBundle { @@ -15,6 +16,15 @@ pub struct KeyBundle { mac_key: Vec, } +fn bytes_to_hex(bytes: &[u8]) -> String { + let mut result = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + // There's no way for this unwrap not to work. + write!(&mut result, "{:02x}", byte).unwrap(); + } + result +} + impl KeyBundle { /// Construct a key bundle from the already-decoded encrypt and hmac keys. @@ -52,8 +62,8 @@ impl KeyBundle { } pub fn from_base64(enc: &str, mac: &str) -> Result { - let enc_bytes = base64::decode_config(&enc, base64::URL_SAFE_NO_PAD)?; - let mac_bytes = base64::decode_config(&mac, base64::URL_SAFE_NO_PAD)?; + let enc_bytes = base64::decode(&enc)?; + let mac_bytes = base64::decode(&mac)?; KeyBundle::new(enc_bytes.into(), mac_bytes.into()) } @@ -80,6 +90,10 @@ impl KeyBundle { Ok(out) } + pub fn hmac_string(&self, ciphertext: &[u8]) -> Result { + Ok(bytes_to_hex(&self.hmac(ciphertext)?)) + } + pub fn verify_hmac(&self, expected_hmac: &[u8], ciphertext_base64: &str) -> Result { let computed_hmac = self.hmac(ciphertext_base64.as_bytes())?; // I suspect this is unnecessary for our case, but the rust-openssl docs @@ -125,4 +139,73 @@ impl KeyBundle { } } +#[cfg(test)] +mod test { + use super::*; + + static HMAC_B16: &'static str = "b1e6c18ac30deb70236bc0d65a46f7a4dce3b8b0e02cf92182b914e3afa5eebc"; + static IV_B64: &'static str = "GX8L37AAb2FZJMzIoXlX8w=="; + static HMAC_KEY_B64: &'static str = "MMntEfutgLTc8FlTLQFms8/xMPmCldqPlq/QQXEjx70="; + static ENC_KEY_B64: &'static str ="9K/wLdXdw+nrTtXo4ZpECyHFNr4d7aYHqeg3KW9+m6Q="; + + static CIPHERTEXT_B64_PIECES: &'static [&'static str] = &[ + "NMsdnRulLwQsVcwxKW9XwaUe7ouJk5Wn80QhbD80l0HEcZGCynh45qIbeYBik0lgcHbK", + "mlIxTJNwU+OeqipN+/j7MqhjKOGIlvbpiPQQLC6/ffF2vbzL0nzMUuSyvaQzyGGkSYM2", + "xUFt06aNivoQTvU2GgGmUK6MvadoY38hhW2LCMkoZcNfgCqJ26lO1O0sEO6zHsk3IVz6", + "vsKiJ2Hq6VCo7hu123wNegmujHWQSGyf8JeudZjKzfi0OFRRvvm4QAKyBWf0MgrW1F8S", + "FDnVfkq8amCB7NhdwhgLWbN+21NitNwWYknoEWe1m6hmGZDgDT32uxzWxCV8QqqrpH/Z", + "ggViEr9uMgoy4lYaWqP7G5WKvvechc62aqnsNEYhH26A5QgzmlNyvB+KPFvPsYzxDnSC", + "jOoRSLx7GG86wT59QZw=" + ]; + + static CLEARTEXT_B64_PIECES: &'static [&'static str] = &[ + "eyJpZCI6IjVxUnNnWFdSSlpYciIsImhpc3RVcmkiOiJmaWxlOi8vL1VzZXJzL2phc29u", + "L0xpYnJhcnkvQXBwbGljYXRpb24lMjBTdXBwb3J0L0ZpcmVmb3gvUHJvZmlsZXMva3Nn", + "ZDd3cGsuTG9jYWxTeW5jU2VydmVyL3dlYXZlL2xvZ3MvIiwidGl0bGUiOiJJbmRleCBv", + "ZiBmaWxlOi8vL1VzZXJzL2phc29uL0xpYnJhcnkvQXBwbGljYXRpb24gU3VwcG9ydC9G", + "aXJlZm94L1Byb2ZpbGVzL2tzZ2Q3d3BrLkxvY2FsU3luY1NlcnZlci93ZWF2ZS9sb2dz", + "LyIsInZpc2l0cyI6W3siZGF0ZSI6MTMxOTE0OTAxMjM3MjQyNSwidHlwZSI6MX1dfQ==" + ]; + + #[test] + fn test_hmac() { + let key_bundle = KeyBundle::from_base64(ENC_KEY_B64, HMAC_KEY_B64).unwrap(); + let ciphertext_base64 = CIPHERTEXT_B64_PIECES.join(""); + let hmac = key_bundle.hmac_string(ciphertext_base64.as_bytes()).unwrap(); + assert_eq!(hmac, HMAC_B16); + } + + #[test] + fn test_decrypt() { + let key_bundle = KeyBundle::from_base64(ENC_KEY_B64, HMAC_KEY_B64).unwrap(); + let ciphertext = base64::decode(&CIPHERTEXT_B64_PIECES.join("")).unwrap(); + let iv = base64::decode(IV_B64).unwrap(); + let s = key_bundle.decrypt(&ciphertext, &iv).unwrap(); + + let cleartext = String::from_utf8( + base64::decode(&CLEARTEXT_B64_PIECES.join("")).unwrap()).unwrap(); + assert_eq!(&cleartext, &s); + } + + #[test] + fn test_encrypt() { + let key_bundle = KeyBundle::from_base64(ENC_KEY_B64, HMAC_KEY_B64).unwrap(); + let iv = base64::decode(IV_B64).unwrap(); + let cleartext_bytes = base64::decode(&CLEARTEXT_B64_PIECES.join("")).unwrap(); + let encrypted_bytes = key_bundle.encrypt_bytes_with_iv(&cleartext_bytes, &iv).unwrap(); + + let expect_ciphertext = base64::decode(&CIPHERTEXT_B64_PIECES.join("")).unwrap(); + + assert_eq!(&encrypted_bytes, &expect_ciphertext); + + + let (enc_bytes2, iv2) = key_bundle.encrypt_bytes_rand_iv(&cleartext_bytes).unwrap(); + + assert_ne!(&enc_bytes2, &expect_ciphertext); + + let s = key_bundle.decrypt(&enc_bytes2, &iv2).unwrap(); + + assert_eq!(&cleartext_bytes, &s.as_bytes()); + } +} From 0461cdaf199802f84b7c4d4ff39d15e7502abfdc Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 23 Apr 2018 15:00:16 -0700 Subject: [PATCH 06/23] Simplify how BsoRecords are deserialized --- sync15-adapter/src/bso_record.rs | 75 ++++++++++++++++-------------- sync15-adapter/src/error.rs | 7 +++ sync15-adapter/src/key_bundle.rs | 10 ++-- sync15-adapter/src/record_types.rs | 38 +++++++++++++++ 4 files changed, 93 insertions(+), 37 deletions(-) create mode 100644 sync15-adapter/src/record_types.rs diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index b872ab8948..85249a73b4 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -5,13 +5,16 @@ use serde::de::{self, Deserialize, Deserializer, Visitor}; use serde::ser::{self, Serialize, Serializer, SerializeStruct}; use serde_json; +use error; +use base64; +use key_bundle::KeyBundle; -use std::collections::HashMap; +// use std::collections::HashMap; use std::marker::PhantomData; use std::fmt; -#[derive(Debug, Clone, Deserialize)] -pub struct BsoRecord where for<'a> T: Deserialize<'a> { +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct BsoRecord where { pub id: String, pub collection: Option, @@ -24,7 +27,7 @@ pub struct BsoRecord where for<'a> T: Deserialize<'a> { // it's payload field as JSON (Especially since this one is going to exist more-or-less just so // that we can decrypt the data... #[serde(deserialize_with = "deserialize_json")] - pub payload: T, + pub payload: serde_json::Value, } #[derive(Deserialize, Serialize, Clone, Debug)] @@ -35,26 +38,6 @@ pub struct EncryptedPayload { pub ciphertext: String, } -#[derive(Deserialize, Serialize, Clone, Debug)] -pub struct MetaGlobalEngine { - pub version: usize, - #[serde(rename = "syncID")] - pub sync_id: String, -} - -#[derive(Deserialize, Serialize, Clone, Debug)] -pub struct MetaGlobalPayload { - #[serde(rename = "syncID")] - pub sync_id: String, - #[serde(rename = "storageVersion")] - pub storage_version: usize, - pub declined: Vec, - pub engines: HashMap, -} - -pub type EncryptedRecord = BsoRecord; -pub type MetaGlobalRecord = BsoRecord; - // Custom deserializer to handle auto-deserializing the payload from JSON. fn deserialize_json<'de, T, D>(deserializer: D) -> Result where for <'a> T: Deserialize<'a>, D: Deserializer<'de> { struct DeserializeNestedJson(PhantomData T>); @@ -76,7 +59,7 @@ fn deserialize_json<'de, T, D>(deserializer: D) -> Result where for } // Custom serializer to handle auto-serializing the payload to JSON -impl Serialize for BsoRecord where T: Serialize, for<'a> T: Deserialize<'a> { +impl Serialize for BsoRecord { fn serialize(&self, serializer: S) -> Result where S: Serializer { // Serialize the object we hold in our payload to a string right away. let payload_json = serde_json::to_string(&self.payload).map_err(|e| ser::Error::custom(e))?; @@ -106,6 +89,30 @@ impl Serialize for BsoRecord where T: Serialize, for<'a> T: Deserialize<'a } } +impl BsoRecord { + pub fn decrypt(&mut self, key: &KeyBundle) -> error::Result<()> { + assert!(self.is_encrypted()); + let payload_data: EncryptedPayload = serde_json::from_value(self.payload.clone())?; + if !key.verify_hmac_string(&payload_data.hmac, &payload_data.ciphertext)? { + return Err(error::ErrorKind::HmacMismatch.into()); + } + + let iv = base64::decode(&payload_data.iv)?; + let ciphertext = base64::decode(&payload_data.ciphertext)?; + let cleartext = key.decrypt(&ciphertext, &iv)?; + + self.payload = serde_json::to_value(cleartext)?; + Ok(()) + } + + pub fn is_encrypted(&self) -> bool { + if let Some(map) = self.payload.as_object() { + map.contains_key("IV") && map.contains_key("hmac") && map.contains_key("ciphertext") + } else { + false + } + } +} #[cfg(test)] @@ -119,29 +126,29 @@ mod tests { "modified": 12344321.0, "payload": "{\"IV\": \"aaaaa\", \"hmac\": \"bbbbb\", \"ciphertext\": \"ccccc\"}" }"#; - let record: EncryptedRecord = serde_json::from_str(serialized).unwrap(); + let record: BsoRecord = serde_json::from_str(serialized).unwrap(); assert_eq!(&record.id, "1234"); assert_eq!(&record.collection.unwrap(), "passwords"); assert_eq!(record.modified, 12344321.0); - assert_eq!(&record.payload.iv, "aaaaa"); - assert_eq!(&record.payload.hmac, "bbbbb"); - assert_eq!(&record.payload.ciphertext, "ccccc"); + let payload: EncryptedPayload = serde_json::from_value(record.payload).unwrap(); + assert_eq!(&payload.iv, "aaaaa"); + assert_eq!(&payload.hmac, "bbbbb"); + assert_eq!(&payload.ciphertext, "ccccc"); } #[test] fn test_serialize_enc() { - // This is sensitive to the order that the fields appear in in EncryptedPayload, unfortunately - let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"hmac\":\"bbbbb\",\"ciphertext\":\"ccccc\"}","collection":"passwords"}"#; - let record = EncryptedRecord { + let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"ciphertext\":\"ccccc\",\"hmac\":\"bbbbb\"}","collection":"passwords"}"#; + let record = BsoRecord { id: "1234".into(), modified: 999.0, // shouldn't be serialized by client no matter what it's value is collection: Some("passwords".into()), sortindex: None, ttl: None, - payload: EncryptedPayload { + payload: serde_json::to_value(EncryptedPayload { iv: "aaaaa".into(), hmac: "bbbbb".into(), ciphertext: "ccccc".into(), - } + }).unwrap() }; let actual = serde_json::to_string(&record).unwrap(); assert_eq!(actual, goal); diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index 8a1f8df59f..a67dfe4aa0 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -7,12 +7,19 @@ error_chain! { Base64Decode(::base64::DecodeError); OpensslError(::openssl::error::ErrorStack); BadCleartextUtf8(::std::string::FromUtf8Error); + JsonError(::serde_json::Error); } errors { BadKeyLength(which_key: &'static str, length: usize) { description("Incorrect key length") display("Incorrect key length for key {}: {}", which_key, length) } + // Not including expected and is (like iOS hmac issues, but unlike desktop) just because + // it's a pain and probably not useful most of the time. This isn't a fundamental issue. + HmacMismatch { + description("SHA256 HMAC Mismatch error") + display("SHA256 HMAC Mismatch error") + } } } diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index 1fc3b41f78..02b55e0295 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -101,6 +101,12 @@ impl KeyBundle { Ok(openssl::memcmp::eq(&expected_hmac, &computed_hmac)) } + pub fn verify_hmac_string(&self, expected_hmac: &str, ciphertext_base64: &str) -> Result { + let computed_hmac = self.hmac(ciphertext_base64.as_bytes())?; + let computed_hmac_string = bytes_to_hex(&computed_hmac); + Ok(openssl::memcmp::eq(&expected_hmac.as_bytes(), &computed_hmac_string.as_bytes())) + } + /// Decrypt the provided ciphertext with the given iv, and decodes the /// result as a utf8 string. Important: Caller must check verify_hmac first! pub fn decrypt(&self, ciphertext: &[u8], iv: &[u8]) -> Result { @@ -173,6 +179,7 @@ mod test { let ciphertext_base64 = CIPHERTEXT_B64_PIECES.join(""); let hmac = key_bundle.hmac_string(ciphertext_base64.as_bytes()).unwrap(); assert_eq!(hmac, HMAC_B16); + assert!(key_bundle.verify_hmac_string(HMAC_B16, &ciphertext_base64).unwrap()); } #[test] @@ -199,13 +206,10 @@ mod test { assert_eq!(&encrypted_bytes, &expect_ciphertext); - let (enc_bytes2, iv2) = key_bundle.encrypt_bytes_rand_iv(&cleartext_bytes).unwrap(); - assert_ne!(&enc_bytes2, &expect_ciphertext); let s = key_bundle.decrypt(&enc_bytes2, &iv2).unwrap(); - assert_eq!(&cleartext_bytes, &s.as_bytes()); } } diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs new file mode 100644 index 0000000000..5e8b2c4afc --- /dev/null +++ b/sync15-adapter/src/record_types.rs @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use key_bundle::KeyBundle; +use errors::{ErrorKind, Result}; + +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct PasswordRecord { + id: String, + hostname: Option, + + #[serde(rename = "formSubmitURL")] + form_submit_url: Option, + + #[serde(rename = "httpRealm")] + http_realm: Option, + + username: String, + password: String, + + #[serde(rename = "usernameField")] + #[serde(default = "")] + username_field: String, + + #[serde(rename = "passwordField")] + #[serde(default = "")] + password_field: String, + + #[serde(rename = "timeCreated")] + time_created: i64, + + #[serde(rename = "timePasswordChanged")] + time_password_changed: i64, + +} + + From 88ebe67be724003c97dcdb55ba9bbe4e811dffd6 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 23 Apr 2018 16:45:03 -0700 Subject: [PATCH 07/23] Fix + test decrypt BsoRecord --- sync15-adapter/src/bso_record.rs | 80 +++++++++++++++++++++++++++++- sync15-adapter/src/lib.rs | 3 +- sync15-adapter/src/record_types.rs | 7 ++- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 85249a73b4..4d01c1610f 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -9,7 +9,6 @@ use error; use base64; use key_bundle::KeyBundle; -// use std::collections::HashMap; use std::marker::PhantomData; use std::fmt; @@ -101,7 +100,7 @@ impl BsoRecord { let ciphertext = base64::decode(&payload_data.ciphertext)?; let cleartext = key.decrypt(&ciphertext, &iv)?; - self.payload = serde_json::to_value(cleartext)?; + self.payload = serde_json::from_str(&cleartext)?; Ok(()) } @@ -112,6 +111,46 @@ impl BsoRecord { false } } + + pub fn encrypt(&mut self, key: &KeyBundle) -> error::Result<()> { + assert!(!self.is_encrypted()); + let cleartext = serde_json::to_string(&self.payload)?; + let (enc_bytes, iv) = key.encrypt_bytes_rand_iv(&cleartext.as_bytes())?; + let iv_base64 = base64::encode(&iv); + let enc_base64 = base64::encode(&enc_bytes); + let hmac = key.hmac_string(enc_base64.as_bytes())?; + let encrypted_payload = json!({ + "IV": iv_base64, + "hmac": hmac, + "ciphertext": enc_base64 + }); + self.payload = serde_json::to_value(encrypted_payload)?; + Ok(()) + } + + /// Returns None if we're encrypted and thus don't know + pub fn is_tombstone(&self) -> Option { + if self.is_encrypted() { + return None; + } + if let Some(&serde_json::Value::Bool(true)) = self.payload.get("deleted") { + Some(true) + } else { + Some(false) + } + } + + pub fn decrypt_clone(&self, kb: &KeyBundle) -> error::Result { + let mut clone = self.clone(); + clone.decrypt(&kb)?; + Ok(clone) + } + + pub fn encrypt_clone(&self, kb: &KeyBundle) -> error::Result { + let mut clone = self.clone(); + clone.encrypt(&kb)?; + Ok(clone) + } } @@ -135,6 +174,7 @@ mod tests { assert_eq!(&payload.hmac, "bbbbb"); assert_eq!(&payload.ciphertext, "ccccc"); } + #[test] fn test_serialize_enc() { let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"ciphertext\":\"ccccc\",\"hmac\":\"bbbbb\"}","collection":"passwords"}"#; @@ -153,4 +193,40 @@ mod tests { let actual = serde_json::to_string(&record).unwrap(); assert_eq!(actual, goal); } + + #[test] + fn test_roundtrip_crypt() { + let mut record = BsoRecord { + id: "aaaaaaaaaaaa".into(), + collection: None, + modified: 1234.0, + sortindex: None, + ttl: None, + payload: json!({ + "id": "aaaaaaaaaaaa", + "deleted": true + }) + }; + + assert!(!record.is_encrypted()); + assert!(record.is_tombstone().unwrap()); + + let keybundle = KeyBundle::new_random().unwrap(); + + record.encrypt(&keybundle).unwrap(); + + assert!(record.is_encrypted()); + let encrypted_data: EncryptedPayload = serde_json::from_value(record.payload.clone()).unwrap(); + assert!(keybundle.verify_hmac_string(&encrypted_data.hmac, &encrypted_data.ciphertext).unwrap()); + + record.decrypt(&keybundle).unwrap(); + assert!(!record.is_encrypted()); + + println!("{:?}", record); + println!("{:?}", record.payload.get("deleted")); + + assert!(record.is_tombstone().unwrap()); + + assert_eq!(record.payload["id"], "aaaaaaaaaaaa"); + } } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 6787f8a4be..3265fd5453 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -9,6 +9,7 @@ extern crate openssl; #[macro_use] extern crate serde_derive; +#[macro_use] extern crate serde_json; #[macro_use] @@ -17,7 +18,7 @@ extern crate error_chain; pub mod key_bundle; pub mod error; pub mod bso_record; - +pub mod record_types; diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 5e8b2c4afc..5b2029f64a 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -2,8 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use key_bundle::KeyBundle; -use errors::{ErrorKind, Result}; +// use error::{ErrorKind, Result}; #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct PasswordRecord { @@ -20,11 +19,11 @@ pub struct PasswordRecord { password: String, #[serde(rename = "usernameField")] - #[serde(default = "")] + #[serde(default = "String::new")] username_field: String, #[serde(rename = "passwordField")] - #[serde(default = "")] + #[serde(default = "String::new")] password_field: String, #[serde(rename = "timeCreated")] From 18cc6deff907d8fa9d544180fdf3a243f0fa28ac Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 24 Apr 2018 12:45:01 -0700 Subject: [PATCH 08/23] Firm up some stuff around crypto and logins --- sync15-adapter/src/bso_record.rs | 24 ++++++++++++++++++--- sync15-adapter/src/error.rs | 12 +++++++++-- sync15-adapter/src/record_types.rs | 34 +++++++++++++++++------------- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 4d01c1610f..e92fcb65a7 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -13,7 +13,7 @@ use std::marker::PhantomData; use std::fmt; #[derive(Debug, Clone, Deserialize, PartialEq)] -pub struct BsoRecord where { +pub struct BsoRecord { pub id: String, pub collection: Option, @@ -90,6 +90,9 @@ impl Serialize for BsoRecord { impl BsoRecord { pub fn decrypt(&mut self, key: &KeyBundle) -> error::Result<()> { + if !self.is_encrypted() { + return Err(error::ErrorKind::BsoWrongCryptState(false).into()); + } assert!(self.is_encrypted()); let payload_data: EncryptedPayload = serde_json::from_value(self.payload.clone())?; if !key.verify_hmac_string(&payload_data.hmac, &payload_data.ciphertext)? { @@ -113,7 +116,9 @@ impl BsoRecord { } pub fn encrypt(&mut self, key: &KeyBundle) -> error::Result<()> { - assert!(!self.is_encrypted()); + if self.is_encrypted() { + return Err(error::ErrorKind::BsoWrongCryptState(true).into()); + } let cleartext = serde_json::to_string(&self.payload)?; let (enc_bytes, iv) = key.encrypt_bytes_rand_iv(&cleartext.as_bytes())?; let iv_base64 = base64::encode(&iv); @@ -128,7 +133,8 @@ impl BsoRecord { Ok(()) } - /// Returns None if we're encrypted and thus don't know + /// Returns None if we're encrypted and thus don't know. + // TODO: Should this be a Result now that BsoWrongCryptState is a thing pub fn is_tombstone(&self) -> Option { if self.is_encrypted() { return None; @@ -151,6 +157,18 @@ impl BsoRecord { clone.encrypt(&kb)?; Ok(clone) } + + /// returns None for tombstone records. + pub fn payload_as(&self) -> error::Result> where for<'a> T: Deserialize<'a> { + if self.is_encrypted() { + return Err(error::ErrorKind::BsoWrongCryptState(false).into()); + } + if self.is_tombstone().unwrap() { + return Ok(None); + } + let deserialized = serde_json::from_value(self.payload.clone())?; + Ok(Some(deserialized)) + } } diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index a67dfe4aa0..c96b053f02 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -14,12 +14,20 @@ error_chain! { description("Incorrect key length") display("Incorrect key length for key {}: {}", which_key, length) } - // Not including expected and is (like iOS hmac issues, but unlike desktop) just because - // it's a pain and probably not useful most of the time. This isn't a fundamental issue. + // Not including `expected` and `is`, since they don't seem useful and are inconvenient + // to include. If we decide we want them it's not too bad to include. HmacMismatch { description("SHA256 HMAC Mismatch error") display("SHA256 HMAC Mismatch error") } + + // Used when a BSO should be decrypted but is encrypted, or vice versa. + BsoWrongCryptState(is_decrypted: bool) { + description("BSO in wrong encryption state for operation") + display("Expected {} BSO, but got a(n) {} one", + if *is_decrypted { "encrypted" } else { "decrypted" }, + if *is_decrypted { "decrypted" } else { "encrypted" }) + } } } diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 5b2029f64a..552e2e76fa 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -5,33 +5,37 @@ // use error::{ErrorKind, Result}; #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct PasswordRecord { - id: String, - hostname: Option, + pub id: String, + pub hostname: Option, + // rename_all = "camelCase" by default will do formSubmitUrl, but we can just + // override this one field. #[serde(rename = "formSubmitURL")] - form_submit_url: Option, + pub form_submit_url: Option, - #[serde(rename = "httpRealm")] - http_realm: Option, + pub http_realm: Option, - username: String, - password: String, + #[serde(default = "String::new")] + pub username: String, + + pub password: String, - #[serde(rename = "usernameField")] #[serde(default = "String::new")] - username_field: String, + pub username_field: String, - #[serde(rename = "passwordField")] #[serde(default = "String::new")] - password_field: String, + pub password_field: String, - #[serde(rename = "timeCreated")] - time_created: i64, + pub time_created: i64, + pub time_password_changed: i64, - #[serde(rename = "timePasswordChanged")] - time_password_changed: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub time_last_used: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub times_used: Option, } From ebfa93a2bbc4d6797199b1d322d65b4dce13ca80 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 24 Apr 2018 14:45:59 -0700 Subject: [PATCH 09/23] Figure out how tombstones play into this, fixup deserialization so that we can avoid raw JSON Values for typical use --- sync15-adapter/src/bso_record.rs | 318 ++++++++++++++++++------------- sync15-adapter/src/lib.rs | 1 - 2 files changed, 185 insertions(+), 134 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index e92fcb65a7..a3f14d68c8 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -2,31 +2,70 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use serde::de::{self, Deserialize, Deserializer, Visitor}; -use serde::ser::{self, Serialize, Serializer, SerializeStruct}; +use serde::de::DeserializeOwned; +use serde::ser::Serialize; use serde_json; use error; use base64; use key_bundle::KeyBundle; -use std::marker::PhantomData; -use std::fmt; - -#[derive(Debug, Clone, Deserialize, PartialEq)] -pub struct BsoRecord { +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BsoRecord { pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] pub collection: Option, + + #[serde(skip_serializing)] pub modified: f64, + + #[serde(skip_serializing_if = "Option::is_none")] pub sortindex: Option, + + #[serde(skip_serializing_if = "Option::is_none")] pub ttl: Option, // We do some serde magic here with serde to parse the payload from JSON as we deserialize. // This avoids having a separate intermediate type that only exists so that we can deserialize // it's payload field as JSON (Especially since this one is going to exist more-or-less just so // that we can decrypt the data... - #[serde(deserialize_with = "deserialize_json")] - pub payload: serde_json::Value, + #[serde(with = "as_json", bound( + serialize = "T: Serialize", + deserialize = "T: DeserializeOwned"))] + pub payload: T, +} + +impl BsoRecord { + #[inline] + pub fn with_payload

(self, payload: P) -> BsoRecord

{ + BsoRecord { + id: self.id, + collection: self.collection, + modified: self.modified, + sortindex: self.sortindex, + ttl: self.ttl, + payload: payload, + } + } +} + +// Contains the methods to automatically deserialize the payload to/from json. +mod as_json { + use serde_json; + use serde::de::{self, Deserialize, DeserializeOwned, Deserializer}; + use serde::ser::{self, Serialize, Serializer}; + + pub fn serialize(t: &T, serializer: S) -> Result + where T: Serialize, S: Serializer { + let j = serde_json::to_string(t).map_err(ser::Error::custom)?; + serializer.serialize_str(&j) + } + + pub fn deserialize<'de, T, D>(deserializer: D) -> Result + where T: DeserializeOwned, D: Deserializer<'de> { + let j = String::deserialize(deserializer)?; + serde_json::from_str(&j).map_err(de::Error::custom) + } } #[derive(Deserialize, Serialize, Clone, Debug)] @@ -37,140 +76,127 @@ pub struct EncryptedPayload { pub ciphertext: String, } -// Custom deserializer to handle auto-deserializing the payload from JSON. -fn deserialize_json<'de, T, D>(deserializer: D) -> Result where for <'a> T: Deserialize<'a>, D: Deserializer<'de> { - struct DeserializeNestedJson(PhantomData T>); +/// Marker trait that indicates that something is a sync record type. By not implementing this +/// for EncryptedPayload, we can statically prevent double-encrypting. +pub trait Sync15Record: Clone + DeserializeOwned + Serialize {} - impl<'de, T> Visitor<'de> for DeserializeNestedJson where for<'a> T: Deserialize<'a> { - type Value = T; +impl Sync15Record for serde_json::Value {} - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("The JSON-encoded payload as string") - } +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +#[serde(untagged)] +pub enum MaybeTombstone { + Tombstone { id: String, deleted: bool }, + Record(T) +} + +impl MaybeTombstone { + + #[inline] + pub fn tombstone>(id: R) -> MaybeTombstone { + MaybeTombstone::Tombstone { id: id.into(), deleted: true } + } - fn visit_str(self, value: &str) -> Result where E: de::Error { - serde_json::from_str(&value).map_err(|e| de::Error::custom(e)) + #[inline] + pub fn is_tombstone(&self) -> bool { + match self { + &MaybeTombstone::Record(_) => false, + _ => true } } - let visitor = DeserializeNestedJson(PhantomData); - deserializer.deserialize_str(visitor) -} + #[inline] + pub fn unwrap(self) -> T { + match self { + MaybeTombstone::Record(record) => record, + _ => panic!("called `MaybeTombstone::unwrap()` on a Tombstone!"), + } + } -// Custom serializer to handle auto-serializing the payload to JSON -impl Serialize for BsoRecord { - fn serialize(&self, serializer: S) -> Result where S: Serializer { - // Serialize the object we hold in our payload to a string right away. - let payload_json = serde_json::to_string(&self.payload).map_err(|e| ser::Error::custom(e))?; - - // We always serialize id and payload, and serialize collection, ttl, and sortindex iff. - // they are present. Annoyingly, serialize_struct requires us tell how many we'll serialize - // up-front. - let num_fields = 2 + (self.collection.is_some() as usize) - + (self.ttl.is_some() as usize) - + (self.sortindex.is_some() as usize); - - // Note: The name here doesn't show up in the output. At least, not for JSON. - let mut state = serializer.serialize_struct("BsoRecord", num_fields)?; - state.serialize_field("id", &self.id)?; - state.serialize_field("payload", &payload_json)?; - - if let &Some(ref collection) = &self.collection { - state.serialize_field("collection", collection)?; + #[inline] + pub fn expect(self, msg: &str) -> T { + match self { + MaybeTombstone::Record(record) => record, + _ => panic!("{}", msg), } - if let &Some(ref sortindex) = &self.sortindex { - state.serialize_field("sortindex", sortindex)?; + } + + #[inline] + pub fn ok_or(self, err: E) -> Result { + match self { + MaybeTombstone::Record(record) => Ok(record), + _ => Err(err) } - if let &Some(ref ttl) = &self.ttl { - state.serialize_field("ttl", ttl)?; + } + + #[inline] + pub fn record(self) -> Option { + match self { + MaybeTombstone::Record(record) => Some(record), + _ => None } - state.end() } } -impl BsoRecord { - pub fn decrypt(&mut self, key: &KeyBundle) -> error::Result<()> { - if !self.is_encrypted() { - return Err(error::ErrorKind::BsoWrongCryptState(false).into()); - } - assert!(self.is_encrypted()); - let payload_data: EncryptedPayload = serde_json::from_value(self.payload.clone())?; - if !key.verify_hmac_string(&payload_data.hmac, &payload_data.ciphertext)? { +impl Sync15Record for MaybeTombstone where T: Sync15Record {} + +impl BsoRecord { + pub fn decrypt(self, key: &KeyBundle) -> error::Result>> + where T: DeserializeOwned { + if !key.verify_hmac_string(&self.payload.hmac, &self.payload.ciphertext)? { return Err(error::ErrorKind::HmacMismatch.into()); } - let iv = base64::decode(&payload_data.iv)?; - let ciphertext = base64::decode(&payload_data.ciphertext)?; + let iv = base64::decode(&self.payload.iv)?; + let ciphertext = base64::decode(&self.payload.ciphertext)?; let cleartext = key.decrypt(&ciphertext, &iv)?; - self.payload = serde_json::from_str(&cleartext)?; - Ok(()) - } + let new_payload = serde_json::from_str::>(&cleartext)?; - pub fn is_encrypted(&self) -> bool { - if let Some(map) = self.payload.as_object() { - map.contains_key("IV") && map.contains_key("hmac") && map.contains_key("ciphertext") - } else { - false - } + let result = self.with_payload(new_payload); + Ok(result) } +} - pub fn encrypt(&mut self, key: &KeyBundle) -> error::Result<()> { - if self.is_encrypted() { - return Err(error::ErrorKind::BsoWrongCryptState(true).into()); - } +impl BsoRecord where T: Sync15Record { + pub fn encrypt(self, key: &KeyBundle) -> error::Result> { let cleartext = serde_json::to_string(&self.payload)?; let (enc_bytes, iv) = key.encrypt_bytes_rand_iv(&cleartext.as_bytes())?; let iv_base64 = base64::encode(&iv); let enc_base64 = base64::encode(&enc_bytes); let hmac = key.hmac_string(enc_base64.as_bytes())?; - let encrypted_payload = json!({ - "IV": iv_base64, - "hmac": hmac, - "ciphertext": enc_base64 + let result = self.with_payload(EncryptedPayload { + iv: iv_base64, + hmac: hmac, + ciphertext: enc_base64, }); - self.payload = serde_json::to_value(encrypted_payload)?; - Ok(()) - } - - /// Returns None if we're encrypted and thus don't know. - // TODO: Should this be a Result now that BsoWrongCryptState is a thing - pub fn is_tombstone(&self) -> Option { - if self.is_encrypted() { - return None; - } - if let Some(&serde_json::Value::Bool(true)) = self.payload.get("deleted") { - Some(true) - } else { - Some(false) - } + Ok(result) } +} - pub fn decrypt_clone(&self, kb: &KeyBundle) -> error::Result { - let mut clone = self.clone(); - clone.decrypt(&kb)?; - Ok(clone) +impl BsoRecord> { + #[inline] + pub fn is_tombstone(&self) -> bool { + self.payload.is_tombstone() } - pub fn encrypt_clone(&self, kb: &KeyBundle) -> error::Result { - let mut clone = self.clone(); - clone.encrypt(&kb)?; - Ok(clone) + #[inline] + pub fn with_record(self) -> Option> where T: Clone { + // XXX how to avoid the clone w/o inlining with_payload + match self.payload.clone() { + MaybeTombstone::Tombstone { .. } => None, + MaybeTombstone::Record(record) => Some(self.with_payload(record)) + } } - /// returns None for tombstone records. - pub fn payload_as(&self) -> error::Result> where for<'a> T: Deserialize<'a> { - if self.is_encrypted() { - return Err(error::ErrorKind::BsoWrongCryptState(false).into()); - } - if self.is_tombstone().unwrap() { - return Ok(None); - } - let deserialized = serde_json::from_value(self.payload.clone())?; - Ok(Some(deserialized)) + #[inline] + pub fn unwrap_record(self) -> BsoRecord where T: Clone { + // XXX how to avoid the clone without inlining with_payload + let unwrapped_payload = self.payload.clone().unwrap(); + self.with_payload(unwrapped_payload) } } +pub type MaybeTombstoneRecord = BsoRecord>; #[cfg(test)] mod tests { @@ -183,68 +209,94 @@ mod tests { "modified": 12344321.0, "payload": "{\"IV\": \"aaaaa\", \"hmac\": \"bbbbb\", \"ciphertext\": \"ccccc\"}" }"#; - let record: BsoRecord = serde_json::from_str(serialized).unwrap(); + let record: BsoRecord = serde_json::from_str(serialized).unwrap(); assert_eq!(&record.id, "1234"); assert_eq!(&record.collection.unwrap(), "passwords"); assert_eq!(record.modified, 12344321.0); - let payload: EncryptedPayload = serde_json::from_value(record.payload).unwrap(); - assert_eq!(&payload.iv, "aaaaa"); - assert_eq!(&payload.hmac, "bbbbb"); - assert_eq!(&payload.ciphertext, "ccccc"); + assert_eq!(&record.payload.iv, "aaaaa"); + assert_eq!(&record.payload.hmac, "bbbbb"); + assert_eq!(&record.payload.ciphertext, "ccccc"); } #[test] fn test_serialize_enc() { - let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"ciphertext\":\"ccccc\",\"hmac\":\"bbbbb\"}","collection":"passwords"}"#; + let goal = r#"{"id":"1234","collection":"passwords","payload":"{\"IV\":\"aaaaa\",\"hmac\":\"bbbbb\",\"ciphertext\":\"ccccc\"}"}"#; let record = BsoRecord { id: "1234".into(), modified: 999.0, // shouldn't be serialized by client no matter what it's value is collection: Some("passwords".into()), sortindex: None, ttl: None, - payload: serde_json::to_value(EncryptedPayload { + payload: EncryptedPayload { iv: "aaaaa".into(), hmac: "bbbbb".into(), ciphertext: "ccccc".into(), - }).unwrap() + } }; let actual = serde_json::to_string(&record).unwrap(); assert_eq!(actual, goal); } + #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] + struct DummyRecord { + id: String, + age: i64, + meta: String, + } + + impl Sync15Record for DummyRecord {} + #[test] - fn test_roundtrip_crypt() { - let mut record = BsoRecord { + fn test_roundtrip_crypt_tombstone() { + let orig_record: MaybeTombstoneRecord = BsoRecord { id: "aaaaaaaaaaaa".into(), collection: None, modified: 1234.0, sortindex: None, ttl: None, - payload: json!({ - "id": "aaaaaaaaaaaa", - "deleted": true - }) + payload: MaybeTombstone::tombstone("aaaaaaaaaaaa") }; - assert!(!record.is_encrypted()); - assert!(record.is_tombstone().unwrap()); + assert!(orig_record.is_tombstone()); let keybundle = KeyBundle::new_random().unwrap(); - record.encrypt(&keybundle).unwrap(); + let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); + + assert!(keybundle.verify_hmac_string( + &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); + assert!(decrypted.is_tombstone()); + assert_eq!(decrypted, orig_record); + } + + #[test] + fn test_roundtrip_crypt_record() { + let orig_record: MaybeTombstoneRecord = BsoRecord { + id: "aaaaaaaaaaaa".into(), + collection: None, + modified: 1234.0, + sortindex: None, + ttl: None, + payload: MaybeTombstone::Record(DummyRecord { + id: "aaaaaaaaaaaa".into(), + age: 105, + meta: "data".into() + }) + }; - assert!(record.is_encrypted()); - let encrypted_data: EncryptedPayload = serde_json::from_value(record.payload.clone()).unwrap(); - assert!(keybundle.verify_hmac_string(&encrypted_data.hmac, &encrypted_data.ciphertext).unwrap()); + assert!(!orig_record.is_tombstone()); - record.decrypt(&keybundle).unwrap(); - assert!(!record.is_encrypted()); + let keybundle = KeyBundle::new_random().unwrap(); - println!("{:?}", record); - println!("{:?}", record.payload.get("deleted")); + let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); - assert!(record.is_tombstone().unwrap()); + assert!(keybundle.verify_hmac_string( + &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); - assert_eq!(record.payload["id"], "aaaaaaaaaaaa"); + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); + assert!(!decrypted.is_tombstone()); + assert_eq!(decrypted, orig_record); } } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 3265fd5453..d398e28b45 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -9,7 +9,6 @@ extern crate openssl; #[macro_use] extern crate serde_derive; -#[macro_use] extern crate serde_json; #[macro_use] From b22952ba48832b42d13f8526010e79bbc2a83f32 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 24 Apr 2018 20:55:39 -0700 Subject: [PATCH 10/23] Untested (but compiling, yay WIP) work for getting keys, and the flow for getting records --- sync15-adapter/Cargo.toml | 7 +- sync15-adapter/src/bso_record.rs | 9 +- sync15-adapter/src/error.rs | 27 ++++ sync15-adapter/src/key_bundle.rs | 5 + sync15-adapter/src/lib.rs | 246 ++++++++++++++++++++++++++++++- 5 files changed, 282 insertions(+), 12 deletions(-) diff --git a/sync15-adapter/Cargo.toml b/sync15-adapter/Cargo.toml index 9b3ad1cba3..2e50e0e1ee 100644 --- a/sync15-adapter/Cargo.toml +++ b/sync15-adapter/Cargo.toml @@ -10,9 +10,8 @@ serde_derive = "1.0" serde_json = "1.0" url = "1.6.0" reqwest = "0.8.2" -hex = "0.3.1" -hkdf = "0.4" -sha2 = "0.7.0" error-chain = "0.11" openssl = "0.10" - +hawk = "1.0" +hyper = "0.11" +log = "0.4" diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index a3f14d68c8..c553ba12c2 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -140,8 +140,11 @@ impl MaybeTombstone { impl Sync15Record for MaybeTombstone where T: Sync15Record {} impl BsoRecord { - pub fn decrypt(self, key: &KeyBundle) -> error::Result>> - where T: DeserializeOwned { + pub fn decrypt(self, key: &KeyBundle) -> error::Result>> where T: DeserializeOwned { + Ok(self.decrypt_as::>(key)?) + } + + pub fn decrypt_as(self, key: &KeyBundle) -> error::Result> where T: DeserializeOwned { if !key.verify_hmac_string(&self.payload.hmac, &self.payload.ciphertext)? { return Err(error::ErrorKind::HmacMismatch.into()); } @@ -150,7 +153,7 @@ impl BsoRecord { let ciphertext = base64::decode(&self.payload.ciphertext)?; let cleartext = key.decrypt(&ciphertext, &iv)?; - let new_payload = serde_json::from_str::>(&cleartext)?; + let new_payload = serde_json::from_str::(&cleartext)?; let result = self.with_payload(new_payload); Ok(result) diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index c96b053f02..3dbb750b7f 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -8,6 +8,9 @@ error_chain! { OpensslError(::openssl::error::ErrorStack); BadCleartextUtf8(::std::string::FromUtf8Error); JsonError(::serde_json::Error); + BadUrl(::reqwest::UrlError); + RequestError(::reqwest::Error); + HawkError(::hawk::Error); } errors { BadKeyLength(which_key: &'static str, length: usize) { @@ -28,7 +31,31 @@ error_chain! { if *is_decrypted { "encrypted" } else { "decrypted" }, if *is_decrypted { "decrypted" } else { "encrypted" }) } + + // Error from tokenserver. Ideally we should probably do a better job here... + TokenserverHttpError(code: ::reqwest::StatusCode) { + description("HTTP status {} when requesting a token from the tokenserver") + display("HTTP status {} when requesting a token from the tokenserver", code) + } + + BackoffError(retry_after_secs: f64) { + description("Server requested backoff") + display("Server requested backoff. Retry after {} seconds.", retry_after_secs) + } + + // We should probably get rid of the ones of these that are actually possible, + // but I'd like to get this done rather than spend tons of time worrying about + // the right error types for now (but at the same time, I'd rather not unwrap) + UnexpectedError(message: String) { + description("Unexpected error") + display("Unexpected error: {}", message) + } } } +// Boilerplate helper... +pub fn unexpected(s: S) -> Error where S: Into { + ErrorKind::UnexpectedError(s.into()).into() +} + diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index 02b55e0295..14f07b1be3 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -77,6 +77,11 @@ impl KeyBundle { &self.mac_key } + #[inline] + pub fn to_b64_vec(&self) -> Vec { + vec![base64::encode(&self.enc_key), base64::encode(&self.mac_key)] + } + /// Returns the 32 byte digest by value since it's small enough to be passed /// around cheaply, and easily convertable into a slice or vec if you want. fn hmac(&self, ciphertext: &[u8]) -> Result<[u8; 32]> { diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index d398e28b45..feadaa37d5 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -2,13 +2,23 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +// `error_chain!` can recurse deeply and I guess we're just supposed to live with that... +#![recursion_limit = "1024"] + extern crate serde; extern crate base64; extern crate openssl; +extern crate reqwest; +extern crate hawk; +#[macro_use] +extern crate hyper; #[macro_use] extern crate serde_derive; +#[macro_use] +extern crate log; + extern crate serde_json; #[macro_use] @@ -19,13 +29,239 @@ pub mod error; pub mod bso_record; pub mod record_types; +use std::collections::HashMap; +use std::cell::Cell; +use std::time::{Duration}; + +use key_bundle::KeyBundle; +use reqwest::{ + Client, + Request, + Response, + Url, + header::{Authorization, Bearer, Accept} +}; +use hyper::Method; +use bso_record::{BsoRecord, Sync15Record, MaybeTombstone, EncryptedPayload}; + +header! { (XKeyID, "X-KeyID") => [String] } +header! { (RetryAfter, "Retry-After") => [f64] } + +// Tokenserver's timestamp +header! { (XTimestamp, "X-Timestamp") => [f64] } +// Storage server's timestamp +header! { (XWeaveTimestamp, "X-Weave-Timestamp") => [f64] } + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Sync15ServiceInit { + pub key_id: String, + pub access_token: String, + pub sync_key: String, + pub tokenserver_base_url: String, +} + +#[derive(Deserialize, Clone, Debug, Eq, PartialEq)] +struct TokenserverToken { + pub id: String, + pub key: String, + pub api_endpoint: String, + pub uid: u64, + pub duration: u64, + pub hashed_fxa_uid: Option, +} + +impl TokenserverToken { + pub fn authorization(&self, req: &Request) -> error::Result> { + let url = req.url(); + let path_and_query = match url.query() { + None => url.path().into(), + Some(qs) => format!("{}?{}", url.path(), qs) + }; + + let creds = hawk::Credentials { + id: self.id.clone(), + key: hawk::Key::new(self.key.as_bytes(), &hawk::SHA256), + }; + + let host = url.host_str().ok_or_else(|| + error::unexpected("Tried to authorize bad URL using hawk (no host)"))?; + + let port = url.port_or_known_default().ok_or_else(|| + error::unexpected("Tried to authorize bad URL using hawk (no port)"))?; + + + let header = hawk::RequestBuilder::new( + req.method().as_ref(), host, port, &path_and_query + ).request().make_header(&creds)?; + + Ok(Authorization(format!("Hawk {}", header))) + } +} + +#[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] +struct CryptoKeysRecord { + pub id: String, + pub default: Vec, + pub collection: String, // "crypto", + pub collections: HashMap> +} + +impl Sync15Record for CryptoKeysRecord {} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct CollectionKeys { + pub default: KeyBundle, + pub collections: HashMap +} + +impl CollectionKeys { + pub fn from_bso(record: BsoRecord) -> error::Result { + Ok(CollectionKeys { + default: KeyBundle::from_base64(&record.payload.default[0], &record.payload.default[1])?, + collections: + record.payload.collections + .iter() + .map(|kv| Ok((kv.0.clone(), KeyBundle::from_base64(&kv.1[0], &kv.1[1])?))) + .collect::>>()? + }) + } + + pub fn to_bso(&self) -> BsoRecord { + BsoRecord { + id: "keys".into(), + collection: Some("crypto".into()), + modified: 0.0, // ignored + sortindex: None, + ttl: None, + payload: CryptoKeysRecord { + id: "keys".into(), + collection: "crypto".into(), + default: self.default.to_b64_vec(), + collections: self.collections.iter().map(|kv| + (kv.0.clone(), kv.1.to_b64_vec())).collect() + }, + } + } + + #[inline] + pub fn key_for_collection<'a>(&'a self, collection: &str) -> &'a KeyBundle { + self.collections.get(collection).unwrap_or(&self.default) + } +} + +#[derive(Debug, Clone)] +pub struct Sync15Service { + init_params: Sync15ServiceInit, + tokenserver_url: reqwest::Url, + root_key: KeyBundle, + client: Client, + last_server_time: Cell, + token: Option, + keys: Option, +} + +impl Sync15Service { + pub fn new(init_params: Sync15ServiceInit) -> error::Result { + let url = init_params.tokenserver_base_url.clone() + "/1.0/sync/1.5"; + let root_key = KeyBundle::from_ksync_base64(&init_params.sync_key)?; + let client = Client::builder().timeout(Duration::from_secs(30)).build()?; + Ok(Sync15Service { + tokenserver_url: Url::parse(&url)?, + client, + init_params, + root_key, + last_server_time: Cell::new(0.0), + token: None, + keys: None, + }) + } + + fn fetch_token(&self) -> error::Result { + let mut resp = self.client + .get(self.tokenserver_url.clone()) + .header(Authorization(Bearer { token: self.init_params.access_token.clone() })) + .header(XKeyID(self.init_params.key_id.clone())) + .send()?; + if !resp.status().is_success() { + warn!("Non-success status when fetching token: {}", resp.status()); + trace!(" Response body {}", resp.text().unwrap_or("???".into())); + if let Some(seconds) = resp.headers().get::().map(|h| **h) { + bail!(error::ErrorKind::BackoffError(seconds)); + } + bail!(error::ErrorKind::TokenserverHttpError(resp.status())); + } + if let Some(timestamp) = resp.headers().get::().map(|h| **h) { + self.last_server_time.set(timestamp); + } else { + bail!(error::unexpected("Missing or corrupted X-Timestamp header from token server")); + } + Ok(resp.json()?) + } + + #[inline] + fn authorization(&self, req: &Request) -> error::Result> { + self.get_token()?.authorization(req) + } + + #[inline] + fn authorized(&self, mut req: Request) -> error::Result { + let header = self.authorization(&req)?; + req.headers_mut().set(header); + Ok(req) + } + #[inline] + fn get_token(&self) -> error::Result<&TokenserverToken> { + // TODO: expiration, etc + Ok(self.token.as_ref().ok_or_else(|| error::unexpected("Don't have token."))?) + } -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); + // TODO: probably want a builder-like API to do collection requests (e.g. something + // that occupies roughly the same conceptual role as the Collection class in desktop) + fn storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { + let url = Url::parse(&self.get_token()?.api_endpoint)?.join(relative_path.as_ref())?; + self.authorized(self.client.request(method, url).header(Accept::json()).build()?) + } + + fn make_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { + let resp = self.client.execute(self.storage_request(method, relative_path)?)?; + // TODO: + // - handle keys_resp errors... + // - record x-weave-timestamp + // - handle backoff + // - x-weave-quota? + // - ... almost certainly other things too... + Ok(resp) + } + + pub fn startup(&mut self) -> error::Result<()> { + self.token = Some(self.fetch_token()?); + let mut keys_resp = self.make_storage_request(Method::Get, "storage/crypto/keys")?; + let keys: BsoRecord = keys_resp.json()?; + self.keys = Some(CollectionKeys::from_bso(keys.decrypt_as(&self.root_key)?)?); + // TODO: error handling and also this isn't nearly enough + Ok(()) + } + + pub fn key_for_collection(&self, collection: &str) -> error::Result<&KeyBundle> { + Ok(self.keys.as_ref() + .ok_or_else(|| error::unexpected("Don't have keys (yet?)"))? + .key_for_collection(collection)) + } + + pub fn all_records(&mut self, collection: &str) -> error::Result>> where T: Sync15Record { + let key = self.key_for_collection(collection)?; + let mut resp = self.make_storage_request(Method::Get, format!("storage/{}?full=1", collection))?; + let records: Vec> = resp.json()?; + let mut result = Vec::with_capacity(records.len()); + for record in records { + let decrypted: BsoRecord> = record.decrypt(key)?; + if let Some(record) = decrypted.with_record() { + result.push(record); + } + } + Ok(result) } } + From fec1cf5bd7397f2029ba8e4d39423c6987b36adf Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 25 Apr 2018 13:21:50 -0700 Subject: [PATCH 11/23] Break code out of lib.rs into token.rs and collection_keys.rs. Misc usability improvements also --- sync15-adapter/src/bso_record.rs | 118 ++++++++++++------ sync15-adapter/src/collection_keys.rs | 61 ++++++++++ sync15-adapter/src/key_bundle.rs | 4 +- sync15-adapter/src/lib.rs | 167 ++++---------------------- sync15-adapter/src/token.rs | 127 ++++++++++++++++++++ 5 files changed, 294 insertions(+), 183 deletions(-) create mode 100644 sync15-adapter/src/collection_keys.rs create mode 100644 sync15-adapter/src/token.rs diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index c553ba12c2..81b8e02ea2 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -7,14 +7,18 @@ use serde::ser::Serialize; use serde_json; use error; use base64; +use std::ops::{Deref, DerefMut}; +use std::convert::From; use key_bundle::KeyBundle; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct BsoRecord { pub id: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub collection: Option, + // It's not clear to me if this actually can be empty in practice. + // firefox-ios seems to think it can... + #[serde(default = "String::new")] + pub collection: String, #[serde(skip_serializing)] pub modified: f64, @@ -28,7 +32,7 @@ pub struct BsoRecord { // We do some serde magic here with serde to parse the payload from JSON as we deserialize. // This avoids having a separate intermediate type that only exists so that we can deserialize // it's payload field as JSON (Especially since this one is going to exist more-or-less just so - // that we can decrypt the data... + // that we can decrypt the data...) #[serde(with = "as_json", bound( serialize = "T: Serialize", deserialize = "T: DeserializeOwned"))] @@ -37,18 +41,71 @@ pub struct BsoRecord { impl BsoRecord { #[inline] - pub fn with_payload

(self, payload: P) -> BsoRecord

{ + pub fn map_payload(self, mapper: F) -> BsoRecord

where F: FnOnce(T) -> P { BsoRecord { id: self.id, collection: self.collection, modified: self.modified, sortindex: self.sortindex, ttl: self.ttl, - payload: payload, + payload: mapper(self.payload), + } + } + + #[inline] + pub fn with_payload

(self, payload: P) -> BsoRecord

{ + self.map_payload(|_| payload) + } +} + +/// Marker trait that indicates that something is a sync record type. By not implementing this +/// for EncryptedPayload, we can statically prevent double-encrypting. +pub trait Sync15Record: Clone + DeserializeOwned + Serialize { + fn collection_tag() -> &'static str; + fn record_id(&self) -> &str; +} + +impl From for BsoRecord where T: Sync15Record { + #[inline] + fn from(payload: T) -> BsoRecord { + let id = payload.record_id().into(); + let collection = T::collection_tag().into(); + BsoRecord { + id, collection, payload, + modified: 0.0, + sortindex: None, + ttl: None, } } } +impl BsoRecord> { + /// Helper to improve ergonomics for handling records that might be tombstones. + #[inline] + pub fn transpose(self) -> Option> { + let BsoRecord { id, collection, modified, sortindex, ttl, payload } = self; + match payload { + Some(p) => Some(BsoRecord { id, collection, modified, sortindex, ttl, payload: p }), + None => None + } + } +} + +impl Deref for BsoRecord { + type Target = T; + #[inline] + fn deref(&self) -> &T { + &self.payload + } +} + +impl DerefMut for BsoRecord { + #[inline] + fn deref_mut(&mut self) -> &mut T { + &mut self.payload + } +} + // Contains the methods to automatically deserialize the payload to/from json. mod as_json { use serde_json; @@ -76,12 +133,6 @@ pub struct EncryptedPayload { pub ciphertext: String, } -/// Marker trait that indicates that something is a sync record type. By not implementing this -/// for EncryptedPayload, we can statically prevent double-encrypting. -pub trait Sync15Record: Clone + DeserializeOwned + Serialize {} - -impl Sync15Record for serde_json::Value {} - #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] #[serde(untagged)] pub enum MaybeTombstone { @@ -90,7 +141,6 @@ pub enum MaybeTombstone { } impl MaybeTombstone { - #[inline] pub fn tombstone>(id: R) -> MaybeTombstone { MaybeTombstone::Tombstone { id: id.into(), deleted: true } @@ -137,14 +187,18 @@ impl MaybeTombstone { } } -impl Sync15Record for MaybeTombstone where T: Sync15Record {} - -impl BsoRecord { - pub fn decrypt(self, key: &KeyBundle) -> error::Result>> where T: DeserializeOwned { - Ok(self.decrypt_as::>(key)?) +impl Sync15Record for MaybeTombstone where T: Sync15Record { + fn collection_tag() -> &'static str { T::collection_tag() } + fn record_id(&self) -> &str { + match self { + &MaybeTombstone::Tombstone { ref id, .. } => id, + &MaybeTombstone::Record(ref record) => record.record_id() + } } +} - pub fn decrypt_as(self, key: &KeyBundle) -> error::Result> where T: DeserializeOwned { +impl BsoRecord { + pub fn decrypt(self, key: &KeyBundle) -> error::Result> where T: DeserializeOwned { if !key.verify_hmac_string(&self.payload.hmac, &self.payload.ciphertext)? { return Err(error::ErrorKind::HmacMismatch.into()); } @@ -183,19 +237,8 @@ impl BsoRecord> { } #[inline] - pub fn with_record(self) -> Option> where T: Clone { - // XXX how to avoid the clone w/o inlining with_payload - match self.payload.clone() { - MaybeTombstone::Tombstone { .. } => None, - MaybeTombstone::Record(record) => Some(self.with_payload(record)) - } - } - - #[inline] - pub fn unwrap_record(self) -> BsoRecord where T: Clone { - // XXX how to avoid the clone without inlining with_payload - let unwrapped_payload = self.payload.clone().unwrap(); - self.with_payload(unwrapped_payload) + pub fn record(self) -> Option> where T: Clone { + self.map_payload(|payload| payload.record()).transpose() } } @@ -214,7 +257,7 @@ mod tests { }"#; let record: BsoRecord = serde_json::from_str(serialized).unwrap(); assert_eq!(&record.id, "1234"); - assert_eq!(&record.collection.unwrap(), "passwords"); + assert_eq!(&record.collection, "passwords"); assert_eq!(record.modified, 12344321.0); assert_eq!(&record.payload.iv, "aaaaa"); assert_eq!(&record.payload.hmac, "bbbbb"); @@ -227,7 +270,7 @@ mod tests { let record = BsoRecord { id: "1234".into(), modified: 999.0, // shouldn't be serialized by client no matter what it's value is - collection: Some("passwords".into()), + collection: "passwords".into(), sortindex: None, ttl: None, payload: EncryptedPayload { @@ -247,13 +290,16 @@ mod tests { meta: String, } - impl Sync15Record for DummyRecord {} + impl Sync15Record for DummyRecord { + fn collection_tag() -> &'static str { "dummy" } + fn record_id(&self) -> &str { &self.id } + } #[test] fn test_roundtrip_crypt_tombstone() { let orig_record: MaybeTombstoneRecord = BsoRecord { id: "aaaaaaaaaaaa".into(), - collection: None, + collection: "dummy".into(), modified: 1234.0, sortindex: None, ttl: None, @@ -278,7 +324,7 @@ mod tests { fn test_roundtrip_crypt_record() { let orig_record: MaybeTombstoneRecord = BsoRecord { id: "aaaaaaaaaaaa".into(), - collection: None, + collection: "dummy".into(), modified: 1234.0, sortindex: None, ttl: None, diff --git a/sync15-adapter/src/collection_keys.rs b/sync15-adapter/src/collection_keys.rs new file mode 100644 index 0000000000..13be35fdfc --- /dev/null +++ b/sync15-adapter/src/collection_keys.rs @@ -0,0 +1,61 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; +use key_bundle::KeyBundle; +use std::collections::HashMap; +use error::Result; + +#[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] +struct CryptoKeysRecord { + pub id: String, + pub collection: String, + pub default: [String; 2], + pub collections: HashMap +} + +impl Sync15Record for CryptoKeysRecord { + fn collection_tag() -> &'static str { "crypto" } + fn record_id(&self) -> &str { "keys" } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CollectionKeys { + pub default: KeyBundle, + pub collections: HashMap +} + +impl CollectionKeys { + pub fn from_encrypted_bso(record: BsoRecord, root_key: &KeyBundle) -> Result { + let keys: BsoRecord = record.decrypt(root_key)?; + Ok(CollectionKeys { + default: KeyBundle::from_base64(&keys.payload.default[0], &keys.payload.default[1])?, + collections: + keys.payload.collections + .into_iter() + .map(|kv| Ok((kv.0, KeyBundle::from_base64(&kv.1[0], &kv.1[1])?))) + .collect::>>()? + }) + } + + fn to_bso(&self) -> BsoRecord { + CryptoKeysRecord { + id: "keys".into(), + collection: "crypto".into(), + default: self.default.to_b64_array(), + collections: self.collections.iter().map(|kv| + (kv.0.clone(), kv.1.to_b64_array())).collect() + }.into() + } + + #[inline] + pub fn to_encrypted_bso(&self, root_key: &KeyBundle) -> Result> { + self.to_bso().encrypt(root_key) + } + + #[inline] + pub fn key_for_collection<'a>(&'a self, collection: &str) -> &'a KeyBundle { + self.collections.get(collection).unwrap_or(&self.default) + } +} diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index 14f07b1be3..28ab1bdd50 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -78,8 +78,8 @@ impl KeyBundle { } #[inline] - pub fn to_b64_vec(&self) -> Vec { - vec![base64::encode(&self.enc_key), base64::encode(&self.mac_key)] + pub fn to_b64_array(&self) -> [String; 2] { + [base64::encode(&self.enc_key), base64::encode(&self.mac_key)] } /// Returns the 32 byte digest by value since it's small enough to be passed diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index feadaa37d5..2e008169dd 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -24,12 +24,14 @@ extern crate serde_json; #[macro_use] extern crate error_chain; +// TODO: Some of these don't need to be pub... pub mod key_bundle; pub mod error; pub mod bso_record; pub mod record_types; +pub mod token; +pub mod collection_keys; -use std::collections::HashMap; use std::cell::Cell; use std::time::{Duration}; @@ -39,16 +41,12 @@ use reqwest::{ Request, Response, Url, - header::{Authorization, Bearer, Accept} + header::Accept }; use hyper::Method; use bso_record::{BsoRecord, Sync15Record, MaybeTombstone, EncryptedPayload}; +use collection_keys::CollectionKeys; -header! { (XKeyID, "X-KeyID") => [String] } -header! { (RetryAfter, "Retry-After") => [f64] } - -// Tokenserver's timestamp -header! { (XTimestamp, "X-Timestamp") => [f64] } // Storage server's timestamp header! { (XWeaveTimestamp, "X-Weave-Timestamp") => [f64] } @@ -60,174 +58,54 @@ pub struct Sync15ServiceInit { pub tokenserver_base_url: String, } -#[derive(Deserialize, Clone, Debug, Eq, PartialEq)] -struct TokenserverToken { - pub id: String, - pub key: String, - pub api_endpoint: String, - pub uid: u64, - pub duration: u64, - pub hashed_fxa_uid: Option, -} - -impl TokenserverToken { - pub fn authorization(&self, req: &Request) -> error::Result> { - let url = req.url(); - let path_and_query = match url.query() { - None => url.path().into(), - Some(qs) => format!("{}?{}", url.path(), qs) - }; - - let creds = hawk::Credentials { - id: self.id.clone(), - key: hawk::Key::new(self.key.as_bytes(), &hawk::SHA256), - }; - - let host = url.host_str().ok_or_else(|| - error::unexpected("Tried to authorize bad URL using hawk (no host)"))?; - - let port = url.port_or_known_default().ok_or_else(|| - error::unexpected("Tried to authorize bad URL using hawk (no port)"))?; - - - let header = hawk::RequestBuilder::new( - req.method().as_ref(), host, port, &path_and_query - ).request().make_header(&creds)?; - - Ok(Authorization(format!("Hawk {}", header))) - } -} - -#[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] -struct CryptoKeysRecord { - pub id: String, - pub default: Vec, - pub collection: String, // "crypto", - pub collections: HashMap> -} - -impl Sync15Record for CryptoKeysRecord {} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct CollectionKeys { - pub default: KeyBundle, - pub collections: HashMap -} - -impl CollectionKeys { - - pub fn from_bso(record: BsoRecord) -> error::Result { - Ok(CollectionKeys { - default: KeyBundle::from_base64(&record.payload.default[0], &record.payload.default[1])?, - collections: - record.payload.collections - .iter() - .map(|kv| Ok((kv.0.clone(), KeyBundle::from_base64(&kv.1[0], &kv.1[1])?))) - .collect::>>()? - }) - } - - pub fn to_bso(&self) -> BsoRecord { - BsoRecord { - id: "keys".into(), - collection: Some("crypto".into()), - modified: 0.0, // ignored - sortindex: None, - ttl: None, - payload: CryptoKeysRecord { - id: "keys".into(), - collection: "crypto".into(), - default: self.default.to_b64_vec(), - collections: self.collections.iter().map(|kv| - (kv.0.clone(), kv.1.to_b64_vec())).collect() - }, - } - } - - #[inline] - pub fn key_for_collection<'a>(&'a self, collection: &str) -> &'a KeyBundle { - self.collections.get(collection).unwrap_or(&self.default) - } -} - -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Sync15Service { init_params: Sync15ServiceInit, - tokenserver_url: reqwest::Url, root_key: KeyBundle, client: Client, last_server_time: Cell, - token: Option, + tsc: token::TokenserverClient, keys: Option, } impl Sync15Service { pub fn new(init_params: Sync15ServiceInit) -> error::Result { - let url = init_params.tokenserver_base_url.clone() + "/1.0/sync/1.5"; let root_key = KeyBundle::from_ksync_base64(&init_params.sync_key)?; let client = Client::builder().timeout(Duration::from_secs(30)).build()?; + // TODO Should we be doing this here? What if we get backoff? Who handles that? + let tsc = token::TokenserverClient::new(&client, + &init_params.tokenserver_base_url, + init_params.access_token.clone(), + init_params.key_id.clone())?; + let timestamp = tsc.server_timestamp(); Ok(Sync15Service { - tokenserver_url: Url::parse(&url)?, client, init_params, root_key, - last_server_time: Cell::new(0.0), - token: None, + tsc, + last_server_time: Cell::new(timestamp), keys: None, }) } - fn fetch_token(&self) -> error::Result { - let mut resp = self.client - .get(self.tokenserver_url.clone()) - .header(Authorization(Bearer { token: self.init_params.access_token.clone() })) - .header(XKeyID(self.init_params.key_id.clone())) - .send()?; - if !resp.status().is_success() { - warn!("Non-success status when fetching token: {}", resp.status()); - trace!(" Response body {}", resp.text().unwrap_or("???".into())); - if let Some(seconds) = resp.headers().get::().map(|h| **h) { - bail!(error::ErrorKind::BackoffError(seconds)); - } - bail!(error::ErrorKind::TokenserverHttpError(resp.status())); - } - if let Some(timestamp) = resp.headers().get::().map(|h| **h) { - self.last_server_time.set(timestamp); - } else { - bail!(error::unexpected("Missing or corrupted X-Timestamp header from token server")); - } - Ok(resp.json()?) - } - - #[inline] - fn authorization(&self, req: &Request) -> error::Result> { - self.get_token()?.authorization(req) - } - #[inline] fn authorized(&self, mut req: Request) -> error::Result { - let header = self.authorization(&req)?; + let header = self.tsc.authorization(&req)?; req.headers_mut().set(header); Ok(req) } - #[inline] - fn get_token(&self) -> error::Result<&TokenserverToken> { - // TODO: expiration, etc - Ok(self.token.as_ref().ok_or_else(|| error::unexpected("Don't have token."))?) - } - // TODO: probably want a builder-like API to do collection requests (e.g. something // that occupies roughly the same conceptual role as the Collection class in desktop) fn storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { - let url = Url::parse(&self.get_token()?.api_endpoint)?.join(relative_path.as_ref())?; + let url = Url::parse(&self.tsc.token().api_endpoint)?.join(relative_path.as_ref())?; self.authorized(self.client.request(method, url).header(Accept::json()).build()?) } fn make_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { let resp = self.client.execute(self.storage_request(method, relative_path)?)?; // TODO: - // - handle keys_resp errors... + // - handle http errors... // - record x-weave-timestamp // - handle backoff // - x-weave-quota? @@ -235,12 +113,11 @@ impl Sync15Service { Ok(resp) } - pub fn startup(&mut self) -> error::Result<()> { - self.token = Some(self.fetch_token()?); + pub fn fetch_keys(&mut self) -> error::Result<()> { let mut keys_resp = self.make_storage_request(Method::Get, "storage/crypto/keys")?; let keys: BsoRecord = keys_resp.json()?; - self.keys = Some(CollectionKeys::from_bso(keys.decrypt_as(&self.root_key)?)?); - // TODO: error handling and also this isn't nearly enough + self.keys = Some(CollectionKeys::from_encrypted_bso(keys, &self.root_key)?); + // TODO: error handling... Ok(()) } @@ -257,7 +134,7 @@ impl Sync15Service { let mut result = Vec::with_capacity(records.len()); for record in records { let decrypted: BsoRecord> = record.decrypt(key)?; - if let Some(record) = decrypted.with_record() { + if let Some(record) = decrypted.record() { result.push(record); } } diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs new file mode 100644 index 0000000000..647fcfd5b9 --- /dev/null +++ b/sync15-adapter/src/token.rs @@ -0,0 +1,127 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use hawk; + +use reqwest::{Client, Request, Url}; +use hyper::header::{Authorization, Bearer}; +use error::{self, Result}; +use std::fmt; +use std::borrow::{Borrow, Cow}; + +/// Tokenserver's timestamp is X-Timestamp and not X-Weave-Timestamp. +header! { (RetryAfter, "Retry-After") => [f64] } + +/// Tokenserver's timestamp is X-Timestamp and not X-Weave-Timestamp. The value is in seconds. +header! { (XTimestamp, "X-Timestamp") => [f64] } + +/// OAuth tokenserver api uses this instead of X-Client-State. +header! { (XKeyID, "X-KeyID") => [String] } + +#[derive(Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct TokenserverToken { + pub id: String, + pub key: String, + pub api_endpoint: String, + pub uid: u64, + pub duration: u64, + // This is treated as optional by at least the desktop client, + // but AFAICT it's always present. + pub hashed_fxa_uid: String, +} + +/// This is really more of a TokenAuthenticator. +pub struct TokenserverClient { + token: TokenserverToken, + server_timestamp: f64, + credentials: hawk::Credentials, +} + +// hawk::Credentials doesn't implement debug -_- +impl fmt::Debug for TokenserverClient { + fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> { + f.debug_struct("TokenserverClient") + .field("token", &self.token) + .field("server_timestamp", &self.server_timestamp) + .field("credentials", &"(omitted)") + .finish() + } +} + +fn token_url(base_url: &str) -> Result { + let mut url = Url::parse(base_url)?; + // kind of gross but avoids problems if base_url has a trailing slash. + url.path_segments_mut() + // We can't do anything anyway if this is the case. + .map_err(|_| error::unexpected("Bad tokenserver url (cannot be base)"))? + .extend(&["1.0", "sync", "1.5"]); + Ok(url) +} + +impl TokenserverClient { + #[inline] + pub fn server_timestamp(&self) -> f64 { + self.server_timestamp + } + + #[inline] + pub fn token(&self) -> &TokenserverToken { + &self.token + } + + pub fn new(request_client: &Client, base_url: &str, access_token: String, key_id: String) -> Result { + let mut resp = request_client.get(token_url(base_url)?) + .header(Authorization(Bearer { token: access_token })) + .header(XKeyID(key_id)) + .send()?; + + if !resp.status().is_success() { + warn!("Non-success status when fetching token: {}", resp.status()); + // TODO: the body should be JSON and contain a status parameter we might need? + trace!(" Response body {}", resp.text().unwrap_or("???".into())); + if let Some(seconds) = resp.headers().get::().map(|h| **h) { + bail!(error::ErrorKind::BackoffError(seconds)); + } + bail!(error::ErrorKind::TokenserverHttpError(resp.status())); + } + + let token: TokenserverToken = resp.json()?; + let timestamp = resp.headers() + .get::() + .map(|h| **h) + .ok_or_else(|| error::unexpected( + "Missing or corrupted X-Timestamp header from token server"))?; + let credentials = hawk::Credentials { + id: token.id.clone(), + key: hawk::Key::new(token.key.as_bytes(), &hawk::SHA256), + }; + Ok(TokenserverClient { token, credentials, server_timestamp: timestamp }) + } + + pub fn authorization(&self, req: &Request) -> Result> { + let url = req.url(); + + let path_and_query = match url.query() { + None => Cow::from(url.path()), + Some(qs) => Cow::from(format!("{}?{}", url.path(), qs)) + }; + + let host = url.host_str().ok_or_else(|| + error::unexpected("Tried to authorize bad URL using hawk (no host)"))?; + + // Known defaults exist for https? (among others), so this should be impossible + let port = url.port_or_known_default().ok_or_else(|| + error::unexpected( + "Tried to authorize bad URL using hawk (no port -- unknown protocol?)"))?; + + let header = hawk::RequestBuilder::new( + req.method().as_ref(), + host, + port, + path_and_query.borrow() + ).request().make_header(&self.credentials)?; + + Ok(Authorization(format!("Hawk {}", header))) + } +} From 4ec94831c7139c5f993b407ada23e62edb698c03 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 25 Apr 2018 13:29:00 -0700 Subject: [PATCH 12/23] Move tombstone stuff into record_types.rs --- sync15-adapter/src/bso_record.rs | 142 -------------------------- sync15-adapter/src/lib.rs | 6 +- sync15-adapter/src/record_types.rs | 155 +++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 144 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 81b8e02ea2..5bf7eb6bc6 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -133,69 +133,6 @@ pub struct EncryptedPayload { pub ciphertext: String, } -#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] -#[serde(untagged)] -pub enum MaybeTombstone { - Tombstone { id: String, deleted: bool }, - Record(T) -} - -impl MaybeTombstone { - #[inline] - pub fn tombstone>(id: R) -> MaybeTombstone { - MaybeTombstone::Tombstone { id: id.into(), deleted: true } - } - - #[inline] - pub fn is_tombstone(&self) -> bool { - match self { - &MaybeTombstone::Record(_) => false, - _ => true - } - } - - #[inline] - pub fn unwrap(self) -> T { - match self { - MaybeTombstone::Record(record) => record, - _ => panic!("called `MaybeTombstone::unwrap()` on a Tombstone!"), - } - } - - #[inline] - pub fn expect(self, msg: &str) -> T { - match self { - MaybeTombstone::Record(record) => record, - _ => panic!("{}", msg), - } - } - - #[inline] - pub fn ok_or(self, err: E) -> Result { - match self { - MaybeTombstone::Record(record) => Ok(record), - _ => Err(err) - } - } - - #[inline] - pub fn record(self) -> Option { - match self { - MaybeTombstone::Record(record) => Some(record), - _ => None - } - } -} - -impl Sync15Record for MaybeTombstone where T: Sync15Record { - fn collection_tag() -> &'static str { T::collection_tag() } - fn record_id(&self) -> &str { - match self { - &MaybeTombstone::Tombstone { ref id, .. } => id, - &MaybeTombstone::Record(ref record) => record.record_id() - } - } -} impl BsoRecord { pub fn decrypt(self, key: &KeyBundle) -> error::Result> where T: DeserializeOwned { @@ -230,20 +167,6 @@ impl BsoRecord where T: Sync15Record { } } -impl BsoRecord> { - #[inline] - pub fn is_tombstone(&self) -> bool { - self.payload.is_tombstone() - } - - #[inline] - pub fn record(self) -> Option> where T: Clone { - self.map_payload(|payload| payload.record()).transpose() - } -} - -pub type MaybeTombstoneRecord = BsoRecord>; - #[cfg(test)] mod tests { use super::*; @@ -283,69 +206,4 @@ mod tests { assert_eq!(actual, goal); } - #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] - struct DummyRecord { - id: String, - age: i64, - meta: String, - } - - impl Sync15Record for DummyRecord { - fn collection_tag() -> &'static str { "dummy" } - fn record_id(&self) -> &str { &self.id } - } - - #[test] - fn test_roundtrip_crypt_tombstone() { - let orig_record: MaybeTombstoneRecord = BsoRecord { - id: "aaaaaaaaaaaa".into(), - collection: "dummy".into(), - modified: 1234.0, - sortindex: None, - ttl: None, - payload: MaybeTombstone::tombstone("aaaaaaaaaaaa") - }; - - assert!(orig_record.is_tombstone()); - - let keybundle = KeyBundle::new_random().unwrap(); - - let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); - - assert!(keybundle.verify_hmac_string( - &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); - - let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); - assert!(decrypted.is_tombstone()); - assert_eq!(decrypted, orig_record); - } - - #[test] - fn test_roundtrip_crypt_record() { - let orig_record: MaybeTombstoneRecord = BsoRecord { - id: "aaaaaaaaaaaa".into(), - collection: "dummy".into(), - modified: 1234.0, - sortindex: None, - ttl: None, - payload: MaybeTombstone::Record(DummyRecord { - id: "aaaaaaaaaaaa".into(), - age: 105, - meta: "data".into() - }) - }; - - assert!(!orig_record.is_tombstone()); - - let keybundle = KeyBundle::new_random().unwrap(); - - let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); - - assert!(keybundle.verify_hmac_string( - &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); - - let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); - assert!(!decrypted.is_tombstone()); - assert_eq!(decrypted, orig_record); - } } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 2e008169dd..0c6a6b4bc2 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -44,7 +44,8 @@ use reqwest::{ header::Accept }; use hyper::Method; -use bso_record::{BsoRecord, Sync15Record, MaybeTombstone, EncryptedPayload}; +use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; +use record_types::MaybeTombstone; use collection_keys::CollectionKeys; // Storage server's timestamp @@ -63,6 +64,7 @@ pub struct Sync15Service { init_params: Sync15ServiceInit, root_key: KeyBundle, client: Client, + // We update this when we make requests last_server_time: Cell, tsc: token::TokenserverClient, keys: Option, @@ -117,7 +119,7 @@ impl Sync15Service { let mut keys_resp = self.make_storage_request(Method::Get, "storage/crypto/keys")?; let keys: BsoRecord = keys_resp.json()?; self.keys = Some(CollectionKeys::from_encrypted_bso(keys, &self.root_key)?); - // TODO: error handling... + // TODO: error handling... key upload? Ok(()) } diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 552e2e76fa..ee7e3994ed 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -3,6 +3,85 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ // use error::{ErrorKind, Result}; +use bso_record::{BsoRecord, Sync15Record}; + +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +#[serde(untagged)] +pub enum MaybeTombstone { + Tombstone { id: String, deleted: bool }, + Record(T) +} + +impl MaybeTombstone { + #[inline] + pub fn tombstone>(id: R) -> MaybeTombstone { + MaybeTombstone::Tombstone { id: id.into(), deleted: true } + } + + #[inline] + pub fn is_tombstone(&self) -> bool { + match self { + &MaybeTombstone::Record(_) => false, + _ => true + } + } + + #[inline] + pub fn unwrap(self) -> T { + match self { + MaybeTombstone::Record(record) => record, + _ => panic!("called `MaybeTombstone::unwrap()` on a Tombstone!"), + } + } + + #[inline] + pub fn expect(self, msg: &str) -> T { + match self { + MaybeTombstone::Record(record) => record, + _ => panic!("{}", msg), + } + } + + #[inline] + pub fn ok_or(self, err: E) -> ::std::result::Result { + match self { + MaybeTombstone::Record(record) => Ok(record), + _ => Err(err) + } + } + + #[inline] + pub fn record(self) -> Option { + match self { + MaybeTombstone::Record(record) => Some(record), + _ => None + } + } +} + +impl Sync15Record for MaybeTombstone where T: Sync15Record { + fn collection_tag() -> &'static str { T::collection_tag() } + fn record_id(&self) -> &str { + match self { + &MaybeTombstone::Tombstone { ref id, .. } => id, + &MaybeTombstone::Record(ref record) => record.record_id() + } + } +} + +impl BsoRecord> { + #[inline] + pub fn is_tombstone(&self) -> bool { + self.payload.is_tombstone() + } + + #[inline] + pub fn record(self) -> Option> where T: Clone { + self.map_payload(|payload| payload.record()).transpose() + } +} + +pub type MaybeTombstoneRecord = BsoRecord>; #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -38,4 +117,80 @@ pub struct PasswordRecord { pub times_used: Option, } +impl Sync15Record for PasswordRecord { + fn collection_tag() -> &'static str { "passwords" } + fn record_id(&self) -> &str { &self.id } +} + +#[cfg(test)] +mod tests { + + use super::*; + use key_bundle::KeyBundle; + + #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] + struct DummyRecord { + id: String, + age: i64, + meta: String, + } + + impl Sync15Record for DummyRecord { + fn collection_tag() -> &'static str { "dummy" } + fn record_id(&self) -> &str { &self.id } + } + + #[test] + fn test_roundtrip_crypt_tombstone() { + let orig_record: MaybeTombstoneRecord = BsoRecord { + id: "aaaaaaaaaaaa".into(), + collection: "dummy".into(), + modified: 1234.0, + sortindex: None, + ttl: None, + payload: MaybeTombstone::tombstone("aaaaaaaaaaaa") + }; + + assert!(orig_record.is_tombstone()); + + let keybundle = KeyBundle::new_random().unwrap(); + + let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); + assert!(keybundle.verify_hmac_string( + &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); + assert!(decrypted.is_tombstone()); + assert_eq!(decrypted, orig_record); + } + + #[test] + fn test_roundtrip_crypt_record() { + let orig_record: MaybeTombstoneRecord = BsoRecord { + id: "aaaaaaaaaaaa".into(), + collection: "dummy".into(), + modified: 1234.0, + sortindex: None, + ttl: None, + payload: MaybeTombstone::Record(DummyRecord { + id: "aaaaaaaaaaaa".into(), + age: 105, + meta: "data".into() + }) + }; + + assert!(!orig_record.is_tombstone()); + + let keybundle = KeyBundle::new_random().unwrap(); + + let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); + + assert!(keybundle.verify_hmac_string( + &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); + assert!(!decrypted.is_tombstone()); + assert_eq!(decrypted, orig_record); + } +} From e77fb59c4ac6b62db07bca559b829577a84102c0 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Thu, 26 Apr 2018 12:57:22 -0700 Subject: [PATCH 13/23] Small refactorings --- sync15-adapter/src/key_bundle.rs | 15 +++------------ sync15-adapter/src/lib.rs | 4 ++++ sync15-adapter/src/record_types.rs | 22 ++++++++++++---------- sync15-adapter/src/util.rs | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 22 deletions(-) create mode 100644 sync15-adapter/src/util.rs diff --git a/sync15-adapter/src/key_bundle.rs b/sync15-adapter/src/key_bundle.rs index 28ab1bdd50..2f8590f703 100644 --- a/sync15-adapter/src/key_bundle.rs +++ b/sync15-adapter/src/key_bundle.rs @@ -3,12 +3,12 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use error::{Result, ErrorKind}; +use util::base16_encode; use base64; use openssl::{self, symm}; use openssl::hash::MessageDigest; use openssl::pkey::PKey; use openssl::sign::Signer; -use std::fmt::Write; #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub struct KeyBundle { @@ -16,15 +16,6 @@ pub struct KeyBundle { mac_key: Vec, } -fn bytes_to_hex(bytes: &[u8]) -> String { - let mut result = String::with_capacity(bytes.len() * 2); - for &byte in bytes { - // There's no way for this unwrap not to work. - write!(&mut result, "{:02x}", byte).unwrap(); - } - result -} - impl KeyBundle { /// Construct a key bundle from the already-decoded encrypt and hmac keys. @@ -96,7 +87,7 @@ impl KeyBundle { } pub fn hmac_string(&self, ciphertext: &[u8]) -> Result { - Ok(bytes_to_hex(&self.hmac(ciphertext)?)) + Ok(base16_encode(&self.hmac(ciphertext)?)) } pub fn verify_hmac(&self, expected_hmac: &[u8], ciphertext_base64: &str) -> Result { @@ -108,7 +99,7 @@ impl KeyBundle { pub fn verify_hmac_string(&self, expected_hmac: &str, ciphertext_base64: &str) -> Result { let computed_hmac = self.hmac(ciphertext_base64.as_bytes())?; - let computed_hmac_string = bytes_to_hex(&computed_hmac); + let computed_hmac_string = base16_encode(&computed_hmac); Ok(openssl::memcmp::eq(&expected_hmac.as_bytes(), &computed_hmac_string.as_bytes())) } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 0c6a6b4bc2..37839742da 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -31,6 +31,10 @@ pub mod bso_record; pub mod record_types; pub mod token; pub mod collection_keys; +pub mod util; + +pub use MaybeTombstone::*; + use std::cell::Cell; use std::time::{Duration}; diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index ee7e3994ed..619671a263 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -5,23 +5,25 @@ // use error::{ErrorKind, Result}; use bso_record::{BsoRecord, Sync15Record}; +pub use MaybeTombstone::*; + #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] #[serde(untagged)] pub enum MaybeTombstone { Tombstone { id: String, deleted: bool }, - Record(T) + NonTombstone(T) } impl MaybeTombstone { #[inline] pub fn tombstone>(id: R) -> MaybeTombstone { - MaybeTombstone::Tombstone { id: id.into(), deleted: true } + Tombstone { id: id.into(), deleted: true } } #[inline] pub fn is_tombstone(&self) -> bool { match self { - &MaybeTombstone::Record(_) => false, + &NonTombstone(_) => false, _ => true } } @@ -29,7 +31,7 @@ impl MaybeTombstone { #[inline] pub fn unwrap(self) -> T { match self { - MaybeTombstone::Record(record) => record, + NonTombstone(record) => record, _ => panic!("called `MaybeTombstone::unwrap()` on a Tombstone!"), } } @@ -37,7 +39,7 @@ impl MaybeTombstone { #[inline] pub fn expect(self, msg: &str) -> T { match self { - MaybeTombstone::Record(record) => record, + NonTombstone(record) => record, _ => panic!("{}", msg), } } @@ -45,7 +47,7 @@ impl MaybeTombstone { #[inline] pub fn ok_or(self, err: E) -> ::std::result::Result { match self { - MaybeTombstone::Record(record) => Ok(record), + NonTombstone(record) => Ok(record), _ => Err(err) } } @@ -53,7 +55,7 @@ impl MaybeTombstone { #[inline] pub fn record(self) -> Option { match self { - MaybeTombstone::Record(record) => Some(record), + NonTombstone(record) => Some(record), _ => None } } @@ -63,8 +65,8 @@ impl Sync15Record for MaybeTombstone where T: Sync15Record { fn collection_tag() -> &'static str { T::collection_tag() } fn record_id(&self) -> &str { match self { - &MaybeTombstone::Tombstone { ref id, .. } => id, - &MaybeTombstone::Record(ref record) => record.record_id() + &Tombstone { ref id, .. } => id, + &NonTombstone(ref record) => record.record_id() } } } @@ -173,7 +175,7 @@ mod tests { modified: 1234.0, sortindex: None, ttl: None, - payload: MaybeTombstone::Record(DummyRecord { + payload: NonTombstone(DummyRecord { id: "aaaaaaaaaaaa".into(), age: 105, meta: "data".into() diff --git a/sync15-adapter/src/util.rs b/sync15-adapter/src/util.rs new file mode 100644 index 0000000000..f2c55518e0 --- /dev/null +++ b/sync15-adapter/src/util.rs @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub fn base16_encode(bytes: &[u8]) -> String { + // This seems to be the fastest way of doing this without using a bunch of unsafe: + // https://gist.github.com/thomcc/c4860d68cf31f9b0283c692f83a239f3 + static HEX_CHARS: &'static [u8] = b"0123456789abcdef"; + let mut result = vec![0u8; bytes.len() * 2]; + let mut index = 0; + for &byte in bytes { + result[index + 0] = HEX_CHARS[(byte >> 4) as usize]; + result[index + 1] = HEX_CHARS[(byte & 15) as usize]; + index += 2; + } + // We know statically that this unwrap is safe, since we can only write ascii + String::from_utf8(result).unwrap() +} From 0b148c5421e3b7f5b2dcad0cf354221c54fddb76 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Thu, 26 Apr 2018 13:08:32 -0700 Subject: [PATCH 14/23] Drop fxa-rust-client from workspace for now because it breaks the build --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 48b36e3643..361b6a7815 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace] members = [ "boxlocker", - "fxa-rust-client", "sync15-adapter" ] From 8b12dc4d4488a6efe898325065858b949c118612 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Thu, 26 Apr 2018 13:47:12 -0700 Subject: [PATCH 15/23] Boxlocker-parity example (that doesn't quite work) --- .gitignore | 1 + sync15-adapter/Cargo.toml | 1 + sync15-adapter/examples/boxlocker-parity.rs | 93 +++++++++++++++++++++ sync15-adapter/src/token.rs | 2 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 sync15-adapter/examples/boxlocker-parity.rs diff --git a/.gitignore b/.gitignore index 84640e8f2c..a46d50a104 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ website/build target Cargo.lock +credentials.json diff --git a/sync15-adapter/Cargo.toml b/sync15-adapter/Cargo.toml index 2e50e0e1ee..674fbfb0e9 100644 --- a/sync15-adapter/Cargo.toml +++ b/sync15-adapter/Cargo.toml @@ -15,3 +15,4 @@ openssl = "0.10" hawk = "1.0" hyper = "0.11" log = "0.4" +env_logger = "0.5" diff --git a/sync15-adapter/examples/boxlocker-parity.rs b/sync15-adapter/examples/boxlocker-parity.rs new file mode 100644 index 0000000000..a6c72b8079 --- /dev/null +++ b/sync15-adapter/examples/boxlocker-parity.rs @@ -0,0 +1,93 @@ + +extern crate sync15_adapter; +extern crate error_chain; +extern crate url; +extern crate base64; +extern crate reqwest; + +extern crate serde; +#[macro_use] +extern crate serde_derive; +extern crate serde_json; + +extern crate env_logger; + +use std::io::Read; +use std::error::Error; +use std::fs; +use std::process; +use std::collections::HashMap; +use std::time::{SystemTime, UNIX_EPOCH}; +use sync15_adapter as sync; + +#[derive(Debug, Deserialize)] +struct OAuthCredentials { + access_token: String, + refresh_token: String, + keys: HashMap, + expires_in: u64, + auth_at: u64, +} + +#[derive(Debug, Deserialize)] +struct ScopedKeyData { + k: String, + kid: String, + scope: String, +} + +fn do_auth(recur: bool) -> Result> { + match fs::File::open("./credentials.json") { + Err(_) => { + if recur { + panic!("Failed to open credentials 2nd time"); + } + println!("No credentials found, invoking boxlocker.py..."); + process::Command::new("python") + .arg("../boxlocker/boxlocker.py").output() + .expect("Failed to run boxlocker.py"); + return do_auth(true); + }, + Ok(mut file) => { + let mut s = String::new(); + file.read_to_string(&mut s)?; + let creds: OAuthCredentials = serde_json::from_str(&s)?; + let time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + if creds.expires_in + creds.auth_at < time { + println!("Warning, credentials may be stale."); + } + Ok(creds) + } + } +} + +fn start() -> Result<(), Box> { + let oauth_data = do_auth(false)?; + + let scope = &oauth_data.keys["https://identity.mozilla.com/apps/oldsync"]; + + let mut svc = sync::Sync15Service::new( + sync::Sync15ServiceInit { + key_id: scope.kid.clone(), + sync_key: scope.k.clone(), + access_token: oauth_data.access_token.clone(), + tokenserver_base_url: "https://stable.dev.lcip.org/syncserver/token".into(), + } + )?; + + svc.fetch_keys()?; + let passwords = svc.all_records::("passwords")?; + + println!("Found {} passwords", passwords.len()); + + for pw in passwords.iter() { + println!("{:?}", pw.payload); + } + + Ok(()) +} + +fn main() { + env_logger::init(); + start().unwrap(); +} diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs index 647fcfd5b9..54f860778e 100644 --- a/sync15-adapter/src/token.rs +++ b/sync15-adapter/src/token.rs @@ -79,7 +79,7 @@ impl TokenserverClient { if !resp.status().is_success() { warn!("Non-success status when fetching token: {}", resp.status()); // TODO: the body should be JSON and contain a status parameter we might need? - trace!(" Response body {}", resp.text().unwrap_or("???".into())); + debug!(" Response body {}", resp.text().unwrap_or("???".into())); if let Some(seconds) = resp.headers().get::().map(|h| **h) { bail!(error::ErrorKind::BackoffError(seconds)); } From 3ad7ef43c3978ad2ead3d003b6c1f997721e74c4 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Thu, 26 Apr 2018 15:12:31 -0700 Subject: [PATCH 16/23] Work towards doing more correct things during setup --- sync15-adapter/examples/boxlocker-parity.rs | 4 +- sync15-adapter/src/error.rs | 14 +++- sync15-adapter/src/lib.rs | 77 +++++++++++++++++++-- sync15-adapter/src/record_types.rs | 23 ++++++ sync15-adapter/src/token.rs | 8 ++- 5 files changed, 117 insertions(+), 9 deletions(-) diff --git a/sync15-adapter/examples/boxlocker-parity.rs b/sync15-adapter/examples/boxlocker-parity.rs index a6c72b8079..b0e0b015f9 100644 --- a/sync15-adapter/examples/boxlocker-parity.rs +++ b/sync15-adapter/examples/boxlocker-parity.rs @@ -71,11 +71,11 @@ fn start() -> Result<(), Box> { key_id: scope.kid.clone(), sync_key: scope.k.clone(), access_token: oauth_data.access_token.clone(), - tokenserver_base_url: "https://stable.dev.lcip.org/syncserver/token".into(), + tokenserver_base_url: "https://oauth-sync.dev.lcip.org/syncserver/token".into(), } )?; - svc.fetch_keys()?; + svc.remote_setup()?; let passwords = svc.all_records::("passwords")?; println!("Found {} passwords", passwords.len()); diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index 3dbb750b7f..ca09b940c7 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -34,15 +34,27 @@ error_chain! { // Error from tokenserver. Ideally we should probably do a better job here... TokenserverHttpError(code: ::reqwest::StatusCode) { - description("HTTP status {} when requesting a token from the tokenserver") + description("HTTP status when requesting a token from the tokenserver") display("HTTP status {} when requesting a token from the tokenserver", code) } + // As above, but for storage requests + StorageHttpError(code: ::reqwest::StatusCode, method: ::hyper::Method, route: String) { + description("HTTP error status when making a request to storage server") + display("HTTP status {} during a storage {} request to \"{}\"", code, method, route) + } + BackoffError(retry_after_secs: f64) { description("Server requested backoff") display("Server requested backoff. Retry after {} seconds.", retry_after_secs) } + // This might just be a NYI, since IDK if we want to upload this. + NoMetaGlobal { + description("No meta global on server for user") + display("No meta global on server for user") + } + // We should probably get rid of the ones of these that are actually possible, // but I'd like to get this done rather than spend tons of time worrying about // the right error types for now (but at the same time, I'd rather not unwrap) diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 37839742da..c283e6b31d 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -38,6 +38,7 @@ pub use MaybeTombstone::*; use std::cell::Cell; use std::time::{Duration}; +use std::collections::HashMap; use key_bundle::KeyBundle; use reqwest::{ @@ -49,7 +50,7 @@ use reqwest::{ }; use hyper::Method; use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; -use record_types::MaybeTombstone; +use record_types::{MaybeTombstone, MetaGlobalRecord}; use collection_keys::CollectionKeys; // Storage server's timestamp @@ -63,6 +64,29 @@ pub struct Sync15ServiceInit { pub tokenserver_base_url: String, } +#[derive(Deserialize, Debug, Clone)] +pub struct InfoConfiguration { + /// The maximum size in bytes of the overall HTTP request body that will be accepted by the + /// server. + pub max_request_bytes: Option, + /// The maximum number of records that can be uploaded to a collection in a single POST request. + pub max_post_records: Option, + /// The maximum combined size in bytes of the record payloads that can be uploaded to a + /// collection in a single POST request. + pub max_post_bytes: Option, + /// The maximum total number of records that can be uploaded to a collection as part of a + /// batched upload. + pub max_total_records: Option, + /// The maximum total combined size in bytes of the record payloads that can be uploaded to a + /// collection as part of a batched upload. + pub max_total_bytes: Option, + /// The maximum size of an individual BSO payload, in bytes. + pub max_record_payload_bytes: Option, +} + + + + #[derive(Debug)] pub struct Sync15Service { init_params: Sync15ServiceInit, @@ -71,7 +95,9 @@ pub struct Sync15Service { // We update this when we make requests last_server_time: Cell, tsc: token::TokenserverClient, + keys: Option, + server_config: Option, } impl Sync15Service { @@ -91,6 +117,7 @@ impl Sync15Service { tsc, last_server_time: Cell::new(timestamp), keys: None, + server_config: None, }) } @@ -109,17 +136,57 @@ impl Sync15Service { } fn make_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { - let resp = self.client.execute(self.storage_request(method, relative_path)?)?; + // I'm shocked that method isn't Copy... + let resp = self.client.execute(self.storage_request(method.clone(), relative_path.as_ref())?)?; + + if let Some(ts) = resp.headers().get::().map(|h| **h) { + self.last_server_time.set(ts); + } else { + // Should we complain more here? + warn!("No X-Weave-Timestamp from storage server!"); + } + + if !resp.status().is_success() { + error!("HTTP error {} ({}) during storage {} to {}", + resp.status().as_u16(), resp.status(), method, relative_path.as_ref()); + bail!(error::ErrorKind::StorageHttpError( + resp.status(), method, relative_path.as_ref().into())); + } + // TODO: - // - handle http errors... - // - record x-weave-timestamp // - handle backoff // - x-weave-quota? // - ... almost certainly other things too... Ok(resp) } - pub fn fetch_keys(&mut self) -> error::Result<()> { + fn fetch_info(&self, path: &str) -> error::Result where for <'a> T: serde::de::Deserialize<'a> { + let mut resp = self.make_storage_request(Method::Get, path)?; + let result: T = resp.json()?; + Ok(result) + } + + pub fn remote_setup(&mut self) -> error::Result<()> { + let server_config = self.fetch_info::("info/configuration")?; + self.server_config = Some(server_config); + let mut resp = match self.make_storage_request(Method::Get, "storage/meta/global") { + Ok(r) => r, + // This is gross, but at least it works. Replace 404s on meta/global with NoMetaGlobal. + Err(error::Error(error::ErrorKind::StorageHttpError(hyper::StatusCode::NotFound, ..), _)) => + bail!(error::ErrorKind::NoMetaGlobal), + Err(e) => return Err(e), + }; + // Note: meta/global is not encrypted! + let meta_global: BsoRecord = resp.json()?; + info!("Meta global: {:?}", meta_global.payload); + let collections = self.fetch_info::>("info/collections")?; + self.update_keys(&collections)?; + Ok(()) + } + + fn update_keys(&mut self, _info_collections: &HashMap) -> error::Result<()> { + // TODO: if info/collections says we should, upload keys. + // TODO: This should be handled in collection_keys.rs, which should track modified time, etc. let mut keys_resp = self.make_storage_request(Method::Get, "storage/crypto/keys")?; let keys: BsoRecord = keys_resp.json()?; self.keys = Some(CollectionKeys::from_encrypted_bso(keys, &self.root_key)?); diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 619671a263..87026463ca 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -4,6 +4,7 @@ // use error::{ErrorKind, Result}; use bso_record::{BsoRecord, Sync15Record}; +use std::collections::HashMap; pub use MaybeTombstone::*; @@ -124,6 +125,28 @@ impl Sync15Record for PasswordRecord { fn record_id(&self) -> &str { &self.id } } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct MetaGlobalEngine { + pub version: usize, + #[serde(rename = "syncID")] + pub sync_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct MetaGlobalRecord { + #[serde(rename = "syncID")] + pub sync_id: String, + #[serde(rename = "storageVersion")] + pub storage_version: usize, + pub engines: HashMap, + pub declined: Vec, +} + +impl Sync15Record for MetaGlobalRecord { + fn collection_tag() -> &'static str { "meta" } + fn record_id(&self) -> &str { "global" } +} + #[cfg(test)] mod tests { diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs index 54f860778e..ed679c584f 100644 --- a/sync15-adapter/src/token.rs +++ b/sync15-adapter/src/token.rs @@ -86,7 +86,13 @@ impl TokenserverClient { bail!(error::ErrorKind::TokenserverHttpError(resp.status())); } - let token: TokenserverToken = resp.json()?; + let mut token: TokenserverToken = resp.json()?; + // Add a trailing slash to the api endpoint instead of at each endpoint. This is required + // for the uid not to get dropped by rust's url crate (which wants stuff like + // `Url::parse("http://example.com/foo.html").join("style.css")` to resolve to + // `http://example.com/style.css`, annoyingly. + token.api_endpoint.push('/'); + let timestamp = resp.headers() .get::() .map(|h| **h) From c0a1c41e41134c70c207517d8b7fce3026ae2d2d Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Thu, 26 Apr 2018 19:52:01 -0700 Subject: [PATCH 17/23] More type safe handling of server time stamps, and add an api for collection request query params --- sync15-adapter/src/bso_record.rs | 10 +- sync15-adapter/src/lib.rs | 39 +++++-- sync15-adapter/src/record_types.rs | 6 +- sync15-adapter/src/request.rs | 180 +++++++++++++++++++++++++++++ sync15-adapter/src/token.rs | 18 +-- sync15-adapter/src/util.rs | 75 ++++++++++++ 6 files changed, 299 insertions(+), 29 deletions(-) create mode 100644 sync15-adapter/src/request.rs diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 5bf7eb6bc6..7eabb17035 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -10,6 +10,7 @@ use base64; use std::ops::{Deref, DerefMut}; use std::convert::From; use key_bundle::KeyBundle; +use util::ServerTimestamp; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct BsoRecord { @@ -21,7 +22,7 @@ pub struct BsoRecord { pub collection: String, #[serde(skip_serializing)] - pub modified: f64, + pub modified: ServerTimestamp, #[serde(skip_serializing_if = "Option::is_none")] pub sortindex: Option, @@ -72,7 +73,7 @@ impl From for BsoRecord where T: Sync15Record { let collection = T::collection_tag().into(); BsoRecord { id, collection, payload, - modified: 0.0, + modified: ServerTimestamp(0.0), sortindex: None, ttl: None, } @@ -133,7 +134,6 @@ pub struct EncryptedPayload { pub ciphertext: String, } - impl BsoRecord { pub fn decrypt(self, key: &KeyBundle) -> error::Result> where T: DeserializeOwned { if !key.verify_hmac_string(&self.payload.hmac, &self.payload.ciphertext)? { @@ -181,7 +181,7 @@ mod tests { let record: BsoRecord = serde_json::from_str(serialized).unwrap(); assert_eq!(&record.id, "1234"); assert_eq!(&record.collection, "passwords"); - assert_eq!(record.modified, 12344321.0); + assert_eq!(record.modified.0, 12344321.0); assert_eq!(&record.payload.iv, "aaaaa"); assert_eq!(&record.payload.hmac, "bbbbb"); assert_eq!(&record.payload.ciphertext, "ccccc"); @@ -192,7 +192,7 @@ mod tests { let goal = r#"{"id":"1234","collection":"passwords","payload":"{\"IV\":\"aaaaa\",\"hmac\":\"bbbbb\",\"ciphertext\":\"ccccc\"}"}"#; let record = BsoRecord { id: "1234".into(), - modified: 999.0, // shouldn't be serialized by client no matter what it's value is + modified: ServerTimestamp(999.0), // shouldn't be serialized by client no matter what it's value is collection: "passwords".into(), sortindex: None, ttl: None, diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index c283e6b31d..0304116c80 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -24,6 +24,8 @@ extern crate serde_json; #[macro_use] extern crate error_chain; +extern crate url; + // TODO: Some of these don't need to be pub... pub mod key_bundle; pub mod error; @@ -32,9 +34,11 @@ pub mod record_types; pub mod token; pub mod collection_keys; pub mod util; +pub mod request; pub use MaybeTombstone::*; +use util::ServerTimestamp; use std::cell::Cell; use std::time::{Duration}; @@ -52,6 +56,7 @@ use hyper::Method; use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; use record_types::{MaybeTombstone, MetaGlobalRecord}; use collection_keys::CollectionKeys; +use request::CollectionRequest; // Storage server's timestamp header! { (XWeaveTimestamp, "X-Weave-Timestamp") => [f64] } @@ -93,7 +98,7 @@ pub struct Sync15Service { root_key: KeyBundle, client: Client, // We update this when we make requests - last_server_time: Cell, + last_server_time: Cell, tsc: token::TokenserverClient, keys: Option, @@ -130,17 +135,22 @@ impl Sync15Service { // TODO: probably want a builder-like API to do collection requests (e.g. something // that occupies roughly the same conceptual role as the Collection class in desktop) - fn storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { - let url = Url::parse(&self.tsc.token().api_endpoint)?.join(relative_path.as_ref())?; + fn build_request(&self, method: Method, url: Url) -> error::Result { self.authorized(self.client.request(method, url).header(Accept::json()).build()?) } - fn make_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { + fn relative_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { + let s = self.tsc.token().api_endpoint.clone() + "/"; + let url = Url::parse(&s)?.join(relative_path.as_ref())?; + Ok(self.make_storage_request(method, url)?) + } + + fn make_storage_request(&self, method: Method, url: Url) -> error::Result { // I'm shocked that method isn't Copy... - let resp = self.client.execute(self.storage_request(method.clone(), relative_path.as_ref())?)?; + let resp = self.client.execute(self.build_request(method.clone(), url)?)?; if let Some(ts) = resp.headers().get::().map(|h| **h) { - self.last_server_time.set(ts); + self.last_server_time.set(ServerTimestamp(ts)); } else { // Should we complain more here? warn!("No X-Weave-Timestamp from storage server!"); @@ -148,9 +158,9 @@ impl Sync15Service { if !resp.status().is_success() { error!("HTTP error {} ({}) during storage {} to {}", - resp.status().as_u16(), resp.status(), method, relative_path.as_ref()); + resp.status().as_u16(), resp.status(), method, resp.url().path()); bail!(error::ErrorKind::StorageHttpError( - resp.status(), method, relative_path.as_ref().into())); + resp.status(), method, resp.url().path().into())); } // TODO: @@ -160,8 +170,13 @@ impl Sync15Service { Ok(resp) } + fn collection_request(&self, method: Method, r: &CollectionRequest) -> error::Result { + self.make_storage_request(method.clone(), + r.build_url(Url::parse(&self.tsc.token().api_endpoint)?)?) + } + fn fetch_info(&self, path: &str) -> error::Result where for <'a> T: serde::de::Deserialize<'a> { - let mut resp = self.make_storage_request(Method::Get, path)?; + let mut resp = self.relative_storage_request(Method::Get, path)?; let result: T = resp.json()?; Ok(result) } @@ -169,7 +184,7 @@ impl Sync15Service { pub fn remote_setup(&mut self) -> error::Result<()> { let server_config = self.fetch_info::("info/configuration")?; self.server_config = Some(server_config); - let mut resp = match self.make_storage_request(Method::Get, "storage/meta/global") { + let mut resp = match self.relative_storage_request(Method::Get, "storage/meta/global") { Ok(r) => r, // This is gross, but at least it works. Replace 404s on meta/global with NoMetaGlobal. Err(error::Error(error::ErrorKind::StorageHttpError(hyper::StatusCode::NotFound, ..), _)) => @@ -187,7 +202,7 @@ impl Sync15Service { fn update_keys(&mut self, _info_collections: &HashMap) -> error::Result<()> { // TODO: if info/collections says we should, upload keys. // TODO: This should be handled in collection_keys.rs, which should track modified time, etc. - let mut keys_resp = self.make_storage_request(Method::Get, "storage/crypto/keys")?; + let mut keys_resp = self.relative_storage_request(Method::Get, "storage/crypto/keys")?; let keys: BsoRecord = keys_resp.json()?; self.keys = Some(CollectionKeys::from_encrypted_bso(keys, &self.root_key)?); // TODO: error handling... key upload? @@ -202,7 +217,7 @@ impl Sync15Service { pub fn all_records(&mut self, collection: &str) -> error::Result>> where T: Sync15Record { let key = self.key_for_collection(collection)?; - let mut resp = self.make_storage_request(Method::Get, format!("storage/{}?full=1", collection))?; + let mut resp = self.collection_request(Method::Get, CollectionRequest::new(collection).full())?; let records: Vec> = resp.json()?; let mut result = Vec::with_capacity(records.len()); for record in records { diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 87026463ca..8dc9f39eb5 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -2,7 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -// use error::{ErrorKind, Result}; use bso_record::{BsoRecord, Sync15Record}; use std::collections::HashMap; @@ -152,6 +151,7 @@ mod tests { use super::*; use key_bundle::KeyBundle; + use util::ServerTimestamp; #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] struct DummyRecord { @@ -170,7 +170,7 @@ mod tests { let orig_record: MaybeTombstoneRecord = BsoRecord { id: "aaaaaaaaaaaa".into(), collection: "dummy".into(), - modified: 1234.0, + modified: ServerTimestamp(1234.0), sortindex: None, ttl: None, payload: MaybeTombstone::tombstone("aaaaaaaaaaaa") @@ -195,7 +195,7 @@ mod tests { let orig_record: MaybeTombstoneRecord = BsoRecord { id: "aaaaaaaaaaaa".into(), collection: "dummy".into(), - modified: 1234.0, + modified: ServerTimestamp(1234.0), sortindex: None, ttl: None, payload: NonTombstone(DummyRecord { diff --git a/sync15-adapter/src/request.rs b/sync15-adapter/src/request.rs new file mode 100644 index 0000000000..a3ce0976b1 --- /dev/null +++ b/sync15-adapter/src/request.rs @@ -0,0 +1,180 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use util::ServerTimestamp; + +use std::fmt; +use url::{Url, UrlQuery, form_urlencoded::Serializer}; +use error::{self, Result}; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum RequestOrder { Oldest, Newest, Index } + +impl fmt::Display for RequestOrder { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + &RequestOrder::Oldest => f.write_str("oldest"), + &RequestOrder::Newest => f.write_str("newest"), + &RequestOrder::Index => f.write_str("index") + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CollectionRequest { + pub collection: String, + pub full: bool, + pub ids: Option>, + pub limit: usize, + pub older: Option, + pub newer: Option, + pub order: Option, + pub commit: bool, + pub batch: Option, +} + +impl CollectionRequest { + #[inline] + pub fn new(collection: S) -> CollectionRequest where S: Into { + CollectionRequest { + collection: collection.into(), + full: false, + ids: None, + limit: 0, + older: None, + newer: None, + order: None, + commit: false, + batch: None, + } + } + + #[inline] + pub fn ids(&mut self, v: V) -> &mut CollectionRequest where V: Into> { + self.ids = Some(v.into()); + self + } + + #[inline] + pub fn full(&mut self) -> &mut CollectionRequest { + self.full = true; + self + } + + #[inline] + pub fn older_than(&mut self, ts: ServerTimestamp) -> &mut CollectionRequest { + self.older = Some(ts); + self + } + + #[inline] + pub fn newer_than(&mut self, ts: ServerTimestamp) -> &mut CollectionRequest { + self.newer = Some(ts); + self + } + + #[inline] + pub fn sort_by(&mut self, order: RequestOrder) -> &mut CollectionRequest { + self.order = Some(order); + self + } + + #[inline] + pub fn limit(&mut self, num: usize) -> &mut CollectionRequest { + self.limit = num; + self + } + + #[inline] + pub fn batch(&mut self, batch: S) -> &mut CollectionRequest where S: Into { + self.batch = Some(batch.into()); + self + } + + #[inline] + pub fn batch_start(&mut self) -> &mut CollectionRequest { + self.batch("true") + } + + #[inline] + pub fn commit(&mut self) -> &mut CollectionRequest { + self.commit = true; + self + } + + fn build_query(&self, pairs: &mut Serializer) { + if self.full { + pairs.append_pair("full", "1"); + } + if self.limit > 0 { + pairs.append_pair("limit", &format!("{}", self.limit)); + } + if let &Some(ref ids) = &self.ids { + pairs.append_pair("ids", &ids.join(",")); + } + if let &Some(ref batch) = &self.batch { + pairs.append_pair("batch", &batch); + } + if self.commit { + pairs.append_pair("commit", "true"); + } + if let Some(ts) = self.older { + pairs.append_pair("older", &format!("{}", ts)); + } + if let Some(ts) = self.newer { + pairs.append_pair("newer", &format!("{}", ts)); + } + if let Some(o) = self.order { + pairs.append_pair("sort", &format!("{}", o)); + } + pairs.finish(); + } + + pub fn build_url(&self, mut base_url: Url) -> Result { + base_url.path_segments_mut() + .map_err(|_| error::unexpected("Not base URL??"))? + .extend(&["storage", &self.collection]); + self.build_query(&mut base_url.query_pairs_mut()); + // This is strange but just accessing query_pairs_mut makes you have + // a trailing question mark on your url. I don't think anything bad + // would happen here, but I don't know, and also, it looks dumb so + // I'd rather not have it. + if base_url.query() == Some("") { + base_url.set_query(None); + } + Ok(base_url) + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_url_building() { + let base = Url::parse("https://example.com/sync").unwrap(); + let empty = CollectionRequest::new("foo").build_url(base.clone()).unwrap(); + assert_eq!(empty.as_str(), "https://example.com/sync/storage/foo"); + let batch_start = CollectionRequest::new("bar").batch_start() + .build_url(base.clone()).unwrap(); + assert_eq!(batch_start.as_str(), "https://example.com/sync/storage/bar?batch=true"); + let batch_commit = CollectionRequest::new("asdf").batch("1234abcdefgh").commit() + .build_url(base.clone()) + .unwrap(); + assert_eq!(batch_commit.as_str(), + "https://example.com/sync/storage/asdf?batch=1234abcdefgh&commit=true"); + + let idreq = CollectionRequest::new("wutang").full().ids(vec!["rza".into(), "gza".into()]) + .build_url(base.clone()).unwrap(); + assert_eq!(idreq.as_str(), "https://example.com/sync/storage/wutang?full=1&ids=rza%2Cgza"); + + let complex = CollectionRequest::new("specific").full().limit(10).sort_by(RequestOrder::Oldest) + .older_than(ServerTimestamp(9876.54)) + .newer_than(ServerTimestamp(1234.56)) + .build_url(base.clone()).unwrap(); + assert_eq!(complex.as_str(), + "https://example.com/sync/storage/specific?full=1&limit=10&older=9876.54&newer=1234.56&sort=oldest"); + + } +} diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs index ed679c584f..d763b628e1 100644 --- a/sync15-adapter/src/token.rs +++ b/sync15-adapter/src/token.rs @@ -9,6 +9,7 @@ use hyper::header::{Authorization, Bearer}; use error::{self, Result}; use std::fmt; use std::borrow::{Borrow, Cow}; +use util::ServerTimestamp; /// Tokenserver's timestamp is X-Timestamp and not X-Weave-Timestamp. header! { (RetryAfter, "Retry-After") => [f64] } @@ -34,7 +35,7 @@ pub struct TokenserverToken { /// This is really more of a TokenAuthenticator. pub struct TokenserverClient { token: TokenserverToken, - server_timestamp: f64, + server_timestamp: ServerTimestamp, credentials: hawk::Credentials, } @@ -61,7 +62,7 @@ fn token_url(base_url: &str) -> Result { impl TokenserverClient { #[inline] - pub fn server_timestamp(&self) -> f64 { + pub fn server_timestamp(&self) -> ServerTimestamp { self.server_timestamp } @@ -86,12 +87,7 @@ impl TokenserverClient { bail!(error::ErrorKind::TokenserverHttpError(resp.status())); } - let mut token: TokenserverToken = resp.json()?; - // Add a trailing slash to the api endpoint instead of at each endpoint. This is required - // for the uid not to get dropped by rust's url crate (which wants stuff like - // `Url::parse("http://example.com/foo.html").join("style.css")` to resolve to - // `http://example.com/style.css`, annoyingly. - token.api_endpoint.push('/'); + let token: TokenserverToken = resp.json()?; let timestamp = resp.headers() .get::() @@ -102,7 +98,11 @@ impl TokenserverClient { id: token.id.clone(), key: hawk::Key::new(token.key.as_bytes(), &hawk::SHA256), }; - Ok(TokenserverClient { token, credentials, server_timestamp: timestamp }) + Ok(TokenserverClient { + token, + credentials, + server_timestamp: ServerTimestamp(timestamp) + }) } pub fn authorization(&self, req: &Request) -> Result> { diff --git a/sync15-adapter/src/util.rs b/sync15-adapter/src/util.rs index f2c55518e0..a11a4b6eb6 100644 --- a/sync15-adapter/src/util.rs +++ b/sync15-adapter/src/util.rs @@ -2,6 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use std::convert::From; +use std::time::Duration; +use std::fmt; + pub fn base16_encode(bytes: &[u8]) -> String { // This seems to be the fastest way of doing this without using a bunch of unsafe: // https://gist.github.com/thomcc/c4860d68cf31f9b0283c692f83a239f3 @@ -16,3 +20,74 @@ pub fn base16_encode(bytes: &[u8]) -> String { // We know statically that this unwrap is safe, since we can only write ascii String::from_utf8(result).unwrap() } + +/// Typesafe way to manage server timestamps without accidentally mixing them up with +/// local ones. +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Deserialize, Serialize)] +pub struct ServerTimestamp(pub f64); + +impl From for f64 { + #[inline] + fn from(ts: ServerTimestamp) -> Self { ts.0 } +} + +impl From for ServerTimestamp { + #[inline] + fn from(ts: f64) -> Self { + assert!(ts >= 0.0); + ServerTimestamp(ts) + } +} + +impl fmt::Display for ServerTimestamp { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +pub const SERVER_EPOCH: ServerTimestamp = ServerTimestamp(0.0); + +impl ServerTimestamp { + /// Returns None if `other` is later than `self` (Duration may not represent + /// negative timespans in rust) + #[inline] + pub fn duration_since(self, other: ServerTimestamp) -> Option { + let delta = self.0 - other.0; + if delta < 0.0 { + None + } else { + let secs = delta.floor(); + // We don't want to round here, since it could round up, and + // Duration::new will panic if it rounds up to 1e9 nanoseconds. + let nanos = ((delta - secs) * 1_000_000_000.0).floor() as u32; + Some(Duration::new(secs as u64, nanos)) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_server_timestamp() { + let t0 = ServerTimestamp(10300.15); + let t1 = ServerTimestamp(10100.05); + assert!(t1.duration_since(t0).is_none()); + assert!(t0.duration_since(t1).is_some()); + let dur = t0.duration_since(t1).unwrap(); + assert_eq!(dur.as_secs(), 200); + assert_eq!(dur.subsec_nanos(), 100_000_000); + } + + #[test] + fn test_base16_encode() { + assert_eq!(base16_encode(&[0x01, 0x10, 0x00, 0x00, 0xab, 0xbc, 0xde, 0xff]), + "01100000abbcdeff"); + assert_eq!(base16_encode(&[]), ""); + assert_eq!(base16_encode(&[0, 0, 0, 0]), "00000000"); + assert_eq!(base16_encode(&[0xff, 0xff, 0xff, 0xff]), "ffffffff"); + assert_eq!(base16_encode(&[0x00, 0x01, 0x02, 0x03, 0x0a]), "000102030a"); + assert_eq!(base16_encode(&[0x00, 0x10, 0x20, 0x30, 0xa0]), "00102030a0"); + } +} From 7ca3dca7151f6dfa8617f83d210bb7e505f08243 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 27 Apr 2018 00:34:40 -0700 Subject: [PATCH 18/23] WIP postqueue (compiles, only sort of tested) --- sync15-adapter/Cargo.toml | 3 + sync15-adapter/src/bso_record.rs | 21 ++ sync15-adapter/src/error.rs | 9 +- sync15-adapter/src/lib.rs | 109 +++++---- sync15-adapter/src/record_types.rs | 13 ++ sync15-adapter/src/request.rs | 351 +++++++++++++++++++++++++++-- sync15-adapter/src/token.rs | 4 +- sync15-adapter/src/util.rs | 11 +- 8 files changed, 461 insertions(+), 60 deletions(-) diff --git a/sync15-adapter/Cargo.toml b/sync15-adapter/Cargo.toml index 674fbfb0e9..97f7ca652e 100644 --- a/sync15-adapter/Cargo.toml +++ b/sync15-adapter/Cargo.toml @@ -15,4 +15,7 @@ openssl = "0.10" hawk = "1.0" hyper = "0.11" log = "0.4" +lazy_static = "1.0" + +[dev-dependencies] env_logger = "0.5" diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 7eabb17035..3f21037559 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -134,6 +134,23 @@ pub struct EncryptedPayload { pub ciphertext: String, } +// This is a little cludgey but I couldn't think of another way to have easy deserialization +// without a bunch of wrapper types, while still only serializing a single time in the +// postqueue. +lazy_static! { + // The number of bytes taken up by padding in a EncryptedPayload. + static ref EMPTY_ENCRYPTED_PAYLOAD_SIZE: usize = serde_json::to_string( + &EncryptedPayload { iv: "".into(), hmac: "".into(), ciphertext: "".into() } + ).unwrap().len(); +} + +impl EncryptedPayload { + #[inline] + pub fn serialized_len(&self) -> usize { + (*EMPTY_ENCRYPTED_PAYLOAD_SIZE) + self.ciphertext.len() + self.hmac.len() + self.iv.len() + } +} + impl BsoRecord { pub fn decrypt(self, key: &KeyBundle) -> error::Result> where T: DeserializeOwned { if !key.verify_hmac_string(&self.payload.hmac, &self.payload.ciphertext)? { @@ -204,6 +221,10 @@ mod tests { }; let actual = serde_json::to_string(&record).unwrap(); assert_eq!(actual, goal); + + let val_str_payload: serde_json::Value = serde_json::from_str(goal).unwrap(); + assert_eq!(val_str_payload["payload"].as_str().unwrap().len(), + record.payload.serialized_len()) } } diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index ca09b940c7..55e1f591c2 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -39,9 +39,9 @@ error_chain! { } // As above, but for storage requests - StorageHttpError(code: ::reqwest::StatusCode, method: ::hyper::Method, route: String) { + StorageHttpError(code: ::reqwest::StatusCode, route: String) { description("HTTP error status when making a request to storage server") - display("HTTP status {} during a storage {} request to \"{}\"", code, method, route) + display("HTTP status {} during a storage request to \"{}\"", code, route) } BackoffError(retry_after_secs: f64) { @@ -62,6 +62,11 @@ error_chain! { description("Unexpected error") display("Unexpected error: {}", message) } + + RecordTooLargeError { + description("Record is larger than the maximum size allowed by the server") + display("Record is larger than the maximum size allowed by the server") + } } } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 0304116c80..4e43f50576 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -13,6 +13,9 @@ extern crate hawk; #[macro_use] extern crate hyper; +#[macro_use] +extern crate lazy_static; + #[macro_use] extern crate serde_derive; @@ -50,16 +53,21 @@ use reqwest::{ Request, Response, Url, - header::Accept + header::{self, Accept} }; use hyper::Method; use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; use record_types::{MaybeTombstone, MetaGlobalRecord}; use collection_keys::CollectionKeys; -use request::CollectionRequest; - -// Storage server's timestamp -header! { (XWeaveTimestamp, "X-Weave-Timestamp") => [f64] } +use request::{ + CollectionRequest, + InfoConfiguration, + XWeaveTimestamp, + XIfUnmodifiedSince, + PostResponse, + BatchPoster, + PostQueue +}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Sync15ServiceInit { @@ -69,29 +77,6 @@ pub struct Sync15ServiceInit { pub tokenserver_base_url: String, } -#[derive(Deserialize, Debug, Clone)] -pub struct InfoConfiguration { - /// The maximum size in bytes of the overall HTTP request body that will be accepted by the - /// server. - pub max_request_bytes: Option, - /// The maximum number of records that can be uploaded to a collection in a single POST request. - pub max_post_records: Option, - /// The maximum combined size in bytes of the record payloads that can be uploaded to a - /// collection in a single POST request. - pub max_post_bytes: Option, - /// The maximum total number of records that can be uploaded to a collection as part of a - /// batched upload. - pub max_total_records: Option, - /// The maximum total combined size in bytes of the record payloads that can be uploaded to a - /// collection as part of a batched upload. - pub max_total_bytes: Option, - /// The maximum size of an individual BSO payload, in bytes. - pub max_record_payload_bytes: Option, -} - - - - #[derive(Debug)] pub struct Sync15Service { init_params: Sync15ServiceInit, @@ -103,8 +88,12 @@ pub struct Sync15Service { keys: Option, server_config: Option, + last_sync_remote: HashMap, } + + + impl Sync15Service { pub fn new(init_params: Sync15ServiceInit) -> error::Result { let root_key = KeyBundle::from_ksync_base64(&init_params.sync_key)?; @@ -123,6 +112,7 @@ impl Sync15Service { last_server_time: Cell::new(timestamp), keys: None, server_config: None, + last_sync_remote: HashMap::new(), }) } @@ -147,26 +137,26 @@ impl Sync15Service { fn make_storage_request(&self, method: Method, url: Url) -> error::Result { // I'm shocked that method isn't Copy... - let resp = self.client.execute(self.build_request(method.clone(), url)?)?; + Ok(self.exec_request(self.build_request(method.clone(), url)?, true)?) + } - if let Some(ts) = resp.headers().get::().map(|h| **h) { - self.last_server_time.set(ServerTimestamp(ts)); - } else { - // Should we complain more here? - warn!("No X-Weave-Timestamp from storage server!"); - } + fn exec_request(&self, req: Request, require_success: bool) -> error::Result { + let resp = self.client.execute(req)?; + + self.update_timestamp(resp.headers()); - if !resp.status().is_success() { - error!("HTTP error {} ({}) during storage {} to {}", - resp.status().as_u16(), resp.status(), method, resp.url().path()); + if require_success && !resp.status().is_success() { + error!("HTTP error {} ({}) during storage request to {}", + resp.status().as_u16(), resp.status(), resp.url().path()); bail!(error::ErrorKind::StorageHttpError( - resp.status(), method, resp.url().path().into())); + resp.status(), resp.url().path().into())); } // TODO: // - handle backoff // - x-weave-quota? // - ... almost certainly other things too... + Ok(resp) } @@ -194,12 +184,13 @@ impl Sync15Service { // Note: meta/global is not encrypted! let meta_global: BsoRecord = resp.json()?; info!("Meta global: {:?}", meta_global.payload); - let collections = self.fetch_info::>("info/collections")?; + let collections = self.fetch_info::>("info/collections")?; self.update_keys(&collections)?; + self.last_sync_remote = collections; Ok(()) } - fn update_keys(&mut self, _info_collections: &HashMap) -> error::Result<()> { + fn update_keys(&mut self, _info_collections: &HashMap) -> error::Result<()> { // TODO: if info/collections says we should, upload keys. // TODO: This should be handled in collection_keys.rs, which should track modified time, etc. let mut keys_resp = self.relative_storage_request(Method::Get, "storage/crypto/keys")?; @@ -228,5 +219,41 @@ impl Sync15Service { } Ok(result) } + + fn update_timestamp(&self, hs: &header::Headers) { + if let Some(ts) = hs.get::().map(|h| **h) { + self.last_server_time.set(ts); + } else { + // Should we complain more here? + warn!("No X-Weave-Timestamp from storage server!"); + } + } + + fn new_post_queue<'a, F: FnMut(PostResponse, bool) -> error::Result<()>>(&'a self, coll: &str, on_response: F) + -> error::Result, F>> { + let ts = self.last_sync_remote + .get(coll) + .ok_or_else(|| error::unexpected(format!("Unknown collection {}", coll)))?; + let pw = PostWrapper { svc: self, coll: coll.into() }; + Ok(PostQueue::new(self.server_config.as_ref().unwrap(), *ts, pw, on_response)) + } } +struct PostWrapper<'a> { + svc: &'a Sync15Service, + coll: String, +} + +impl<'a> BatchPoster for PostWrapper<'a> { + fn post(&mut self, bytes: &[u8], xius: ServerTimestamp, batch: Option, commit: bool) -> error::Result { + let url = CollectionRequest::new(self.coll.clone()) + .batch(batch) + .commit(commit) + .build_url(Url::parse(&self.svc.tsc.token().api_endpoint)?)?; + + let mut req = self.svc.build_request(Method::Post, url)?; + req.headers_mut().set(XIfUnmodifiedSince(xius)); + let mut resp = self.svc.exec_request(req, false)?; + Ok(PostResponse::from_response(&mut resp)?) + } +} diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 8dc9f39eb5..9cd4f98d1b 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -152,6 +152,7 @@ mod tests { use super::*; use key_bundle::KeyBundle; use util::ServerTimestamp; + use serde_json; #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] struct DummyRecord { @@ -185,6 +186,12 @@ mod tests { assert!(keybundle.verify_hmac_string( &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + // While we're here, check on EncryptedPayload::serialized_len + let val_rec = serde_json::from_str::( + &serde_json::to_string(&encrypted).unwrap()).unwrap(); + assert_eq!(encrypted.payload.serialized_len(), + val_rec["payload"].as_str().unwrap().len()); + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); assert!(decrypted.is_tombstone()); assert_eq!(decrypted, orig_record); @@ -214,6 +221,12 @@ mod tests { assert!(keybundle.verify_hmac_string( &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + // While we're here, check on EncryptedPayload::serialized_len + let val_rec = serde_json::from_str::( + &serde_json::to_string(&encrypted).unwrap()).unwrap(); + assert_eq!(encrypted.payload.serialized_len(), + val_rec["payload"].as_str().unwrap().len()); + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); assert!(!decrypted.is_tombstone()); assert_eq!(decrypted, orig_record); diff --git a/sync15-adapter/src/request.rs b/sync15-adapter/src/request.rs index a3ce0976b1..b5128bf62a 100644 --- a/sync15-adapter/src/request.rs +++ b/sync15-adapter/src/request.rs @@ -3,14 +3,24 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use util::ServerTimestamp; +use bso_record::{BsoRecord, EncryptedPayload}; -use std::fmt; +use serde_json; +use std::fmt::{self, Write}; +use std::collections::HashMap; use url::{Url, UrlQuery, form_urlencoded::Serializer}; use error::{self, Result}; +use hyper::{StatusCode}; +use reqwest::Response; + #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum RequestOrder { Oldest, Newest, Index } +header! { (XIfUnmodifiedSince, "X-If-Unmodified-Since") => [ServerTimestamp] } +header! { (XLastModified, "X-Last-Modified") => [ServerTimestamp] } +header! { (XWeaveTimestamp, "X-Weave-Timestamp") => [ServerTimestamp] } + impl fmt::Display for RequestOrder { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -88,19 +98,14 @@ impl CollectionRequest { } #[inline] - pub fn batch(&mut self, batch: S) -> &mut CollectionRequest where S: Into { - self.batch = Some(batch.into()); + pub fn batch(&mut self, batch: Option) -> &mut CollectionRequest { + self.batch = batch; self } #[inline] - pub fn batch_start(&mut self) -> &mut CollectionRequest { - self.batch("true") - } - - #[inline] - pub fn commit(&mut self) -> &mut CollectionRequest { - self.commit = true; + pub fn commit(&mut self, v: bool) -> &mut CollectionRequest { + self.commit = v; self } @@ -148,6 +153,324 @@ impl CollectionRequest { } } +/// Manages a pair of (byte, count) limits for a PostQueue, such as +/// (max_post_bytes, max_post_records) or (max_total_bytes, max_total_records). +#[derive(Debug, Clone)] +struct LimitTracker { + max_bytes: usize, + max_records: usize, + cur_bytes: usize, + cur_records: usize, +} + +impl LimitTracker { + pub fn new(max_bytes: usize, max_records: usize) -> LimitTracker { + LimitTracker { + max_bytes, + max_records, + cur_bytes: 0, + cur_records: 0 + } + } + + pub fn clear(&mut self) { + self.cur_records = 0; + self.cur_bytes = 0; + } + + pub fn can_add_record(&self, payload_size: usize) -> bool { + // Desktop does the cur_bytes check as exclusive, but we shouldn't see any servers that + // don't have https://github.com/mozilla-services/server-syncstorage/issues/73 + self.cur_records + 1 <= self.max_records && + self.cur_bytes + payload_size <= self.max_bytes + } + + pub fn can_never_add(&self, record_size: usize) -> bool { + record_size >= self.max_bytes + } + + pub fn record_added(&mut self, record_size: usize) { + assert!(self.can_add_record(record_size), + "LimitTracker::record_added caller must check can_add_record"); + self.cur_records += 1; + self.cur_bytes += 1; + } +} + +#[derive(Deserialize, Debug, Clone)] +pub struct InfoConfiguration { + /// The maximum size in bytes of the overall HTTP request body that will be accepted by the + /// server. + #[serde(default = "default_max_request_bytes")] + pub max_request_bytes: usize, + + /// The maximum number of records that can be uploaded to a collection in a single POST request. + #[serde(default = "usize::max_value")] + pub max_post_records: usize, + + /// The maximum combined size in bytes of the record payloads that can be uploaded to a + /// collection in a single POST request. + #[serde(default = "usize::max_value")] + pub max_post_bytes: usize, + + /// The maximum total number of records that can be uploaded to a collection as part of a + /// batched upload. + #[serde(default = "usize::max_value")] + pub max_total_records: usize, + + /// The maximum total combined size in bytes of the record payloads that can be uploaded to a + /// collection as part of a batched upload. + #[serde(default = "usize::max_value")] + pub max_total_bytes: usize, + + /// The maximum size of an individual BSO payload, in bytes. + #[serde(default = "default_max_record_payload_bytes")] + pub max_record_payload_bytes: usize, +} + +// This is annoying but seems to be the only way to do it. +fn default_max_request_bytes() -> usize { 260 * 1024 } +fn default_max_record_payload_bytes() -> usize { 256 * 1024 } + +#[derive(Debug, Clone, Deserialize)] +pub struct UploadResult { + batch: Option, + /// Maps record id => why failde + pub failed: HashMap, + /// Vec of ids + pub success: Vec +} + +// Easier to fake during tests +#[derive(Debug, Clone)] +pub struct PostResponse { + pub status: StatusCode, + pub result: UploadResult, // This is lazy... + pub last_modified: ServerTimestamp, +} + +impl PostResponse { + pub fn from_response(r: &mut Response) -> Result { + let result: UploadResult = r.json()?; + let last_modified = r.headers().get::().map(|h| **h).ok_or_else(|| + error::unexpected("Server didn't send X-Last-Modified header"))?; + let status = r.status(); + Ok(PostResponse { status, result, last_modified }) + } +} + + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum BatchState { + Unsupported, + NoBatch, + InBatch(String), +} + +#[derive(Debug)] +pub struct PostQueue { + poster: Post, + on_response: OnResponse, + post_limits: LimitTracker, + batch_limits: LimitTracker, + max_payload_bytes: usize, + max_request_bytes: usize, + queued: Vec, + batch: BatchState, + last_modified: ServerTimestamp, +} + +pub trait BatchPoster { + fn post(&mut self, + body: &[u8], + ts: ServerTimestamp, + batch: Option, + commit: bool) -> Result; +} + +/// The Poster param takes +/// 1. A slice that will represent the request body +/// 2. XIUS value for the request +/// 3. An optional batch id, which will be None if batching is not supported. +/// 4. A boolean for whether or not this is a batch commit. +/// +/// Note: Poster should not report non-success HTTP statuses as errors!! +impl PostQueue +where + Poster: BatchPoster, + OnResponse: FnMut(PostResponse, bool) -> Result<()> +{ + pub fn new(config: &InfoConfiguration, ts: ServerTimestamp, poster: Poster, on_response: OnResponse) -> PostQueue { + PostQueue { + poster, + on_response, + last_modified: ts, + post_limits: LimitTracker::new(config.max_post_bytes, config.max_post_records), + batch_limits: LimitTracker::new(config.max_total_bytes, config.max_total_records), + batch: BatchState::NoBatch, + max_payload_bytes: config.max_record_payload_bytes, + max_request_bytes: config.max_request_bytes, + queued: Vec::new(), + } + } + + #[inline] + fn in_batch(&self) -> bool { + match &self.batch { + &BatchState::Unsupported | + &BatchState::NoBatch => true, + _ => false + } + } + + pub fn enqueue(&mut self, record: &BsoRecord) -> Result { + let payload_length = record.payload.serialized_len(); + + if self.post_limits.can_never_add(payload_length) || + self.batch_limits.can_never_add(payload_length) || + payload_length >= self.max_payload_bytes { + warn!("Single record too large to submit to server ({} b)", payload_length); + return Ok(false); + } + + // Write directly into `queued` but undo if necessary (the vast majority of the time + // it won't be necessary). If we hit a problem we need to undo that, but the only error + // case we have to worry about right now is in flush() + let item_start = self.queued.len(); + + // This is conservative but can't hurt. + self.queued.reserve(payload_length + 2); + + // Either the first character in an array, or a comma separating + // it from the previous item. + let c = if self.queued.is_empty() { b'[' } else { b',' }; + self.queued.push(c); + + // This unwrap is fine, since serde_json's failure case is HashMaps that have non-object + // keys, which is impossible. If you decide to change this part, you *need* to call + // `self.queued.truncate(item_start)` here in the failure case! + serde_json::to_writer(&mut self.queued, &record).unwrap(); + + let item_end = self.queued.len(); + + debug_assert!(item_end >= payload_length, + "EncryptedPayload::serialized_len is bugged"); + + // The + 1 is only relevant for the final record, which will have a trailing ']'. + let item_len = item_end - item_start + 1; + + if item_len >= self.max_request_bytes { + self.queued.truncate(item_start); + warn!("Single record too large to submit to server ({} b)", item_len); + return Ok(false); + } + + let can_post_record = self.post_limits.can_add_record(payload_length); + let can_batch_record = self.batch_limits.can_add_record(payload_length); + let can_send_record = self.queued.len() < self.max_request_bytes; + + if !can_post_record || !can_send_record || !can_batch_record { + debug!("PostQueue flushing! (can_post = {}, can_send = {}, can_batch = {})", + can_post_record, can_send_record, can_batch_record); + // "unwrite" the record. + self.queued.truncate(item_start); + // Flush whatever we have queued. + self.flush(!can_batch_record)?; + // And write it again. + let c = if self.queued.is_empty() { b'[' } else { b',' }; + self.queued.push(c); + serde_json::to_writer(&mut self.queued, &record).unwrap(); + } + + self.post_limits.record_added(payload_length); + self.batch_limits.record_added(payload_length); + + Ok(true) + } + + pub fn flush(&mut self, want_commit: bool) -> Result<()> { + if self.queued.len() == 0 { + assert!(!self.in_batch(), + "Bug: Somehow we're in a batch but have no queued records"); + // Nothing to do! + return Ok(()); + } + + self.queued.push(b']'); + let batch_id = match &self.batch { + // Not the first post and we know we have no batch semantics. + &BatchState::Unsupported => None, + // First commit in possible batch + &BatchState::NoBatch => Some("true".into()), + // In a batch and we have a batch id. + &BatchState::InBatch(ref s) => Some(s.clone()) + }; + + info!("Posting {} records of {} bytes", self.post_limits.cur_records, self.queued.len()); + + let is_commit = want_commit && !batch_id.is_none(); + // Weird syntax for calling a function object that is a property. + let resp_or_error = self.poster.post(&self.queued, self.last_modified, batch_id, is_commit); + + self.queued.truncate(0); + + if want_commit { + self.batch_limits.clear(); + } + self.post_limits.clear(); + + let resp = resp_or_error?; + + if !resp.status.is_success() { + (self.on_response)(resp, !want_commit)?; + bail!(error::unexpected("Expected OnResponse to have bailed out!")); + } + + if want_commit { + debug!("Committed batch {:?}", self.batch); + self.batch = BatchState::NoBatch; + self.last_modified = resp.last_modified; + (self.on_response)(resp, false)?; + return Ok(()); + } + + if resp.status != StatusCode::Accepted { + if self.in_batch() { + bail!(error::unexpected( + "Server responded non-202 success code while a batch was in progress")); + } + self.last_modified = resp.last_modified; + self.batch = BatchState::Unsupported; + (self.on_response)(resp, false)?; + return Ok(()); + } + + let batch_id = resp.result.batch.as_ref().ok_or_else(|| + error::unexpected("Invalid server response: 202 without a batch ID"))?.clone(); + + match &self.batch { + &BatchState::Unsupported => { + warn!("Server changed it's mind about supporting batching mid-batch..."); + }, + + &BatchState::InBatch(ref cur_id) => { + if cur_id != &batch_id { + bail!(error::unexpected("Server changed batch id mid-batch!")); + } + }, + _ => {} + } + + // Can't change this in match arms without NLL + self.batch = BatchState::InBatch(batch_id); + self.last_modified = resp.last_modified; + + (self.on_response)(resp, false)?; + + Ok(()) + } +} + #[cfg(test)] mod test { use super::*; @@ -156,17 +479,17 @@ mod test { let base = Url::parse("https://example.com/sync").unwrap(); let empty = CollectionRequest::new("foo").build_url(base.clone()).unwrap(); assert_eq!(empty.as_str(), "https://example.com/sync/storage/foo"); - let batch_start = CollectionRequest::new("bar").batch_start() + let batch_start = CollectionRequest::new("bar").batch(Some("true".into())).commit(false) .build_url(base.clone()).unwrap(); assert_eq!(batch_start.as_str(), "https://example.com/sync/storage/bar?batch=true"); - let batch_commit = CollectionRequest::new("asdf").batch("1234abcdefgh").commit() + let batch_commit = CollectionRequest::new("asdf").batch(Some("1234abc".into())).commit(true) .build_url(base.clone()) .unwrap(); assert_eq!(batch_commit.as_str(), - "https://example.com/sync/storage/asdf?batch=1234abcdefgh&commit=true"); + "https://example.com/sync/storage/asdf?batch=1234abc&commit=true"); let idreq = CollectionRequest::new("wutang").full().ids(vec!["rza".into(), "gza".into()]) - .build_url(base.clone()).unwrap(); + .build_url(base.clone()).unwrap(); assert_eq!(idreq.as_str(), "https://example.com/sync/storage/wutang?full=1&ids=rza%2Cgza"); let complex = CollectionRequest::new("specific").full().limit(10).sort_by(RequestOrder::Oldest) diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs index d763b628e1..88aa436e0d 100644 --- a/sync15-adapter/src/token.rs +++ b/sync15-adapter/src/token.rs @@ -15,7 +15,7 @@ use util::ServerTimestamp; header! { (RetryAfter, "Retry-After") => [f64] } /// Tokenserver's timestamp is X-Timestamp and not X-Weave-Timestamp. The value is in seconds. -header! { (XTimestamp, "X-Timestamp") => [f64] } +header! { (XTimestamp, "X-Timestamp") => [ServerTimestamp] } /// OAuth tokenserver api uses this instead of X-Client-State. header! { (XKeyID, "X-KeyID") => [String] } @@ -101,7 +101,7 @@ impl TokenserverClient { Ok(TokenserverClient { token, credentials, - server_timestamp: ServerTimestamp(timestamp) + server_timestamp: timestamp }) } diff --git a/sync15-adapter/src/util.rs b/sync15-adapter/src/util.rs index a11a4b6eb6..0930f2471e 100644 --- a/sync15-adapter/src/util.rs +++ b/sync15-adapter/src/util.rs @@ -4,7 +4,8 @@ use std::convert::From; use std::time::Duration; -use std::fmt; +use std::{fmt, num}; +use std::str::FromStr; pub fn base16_encode(bytes: &[u8]) -> String { // This seems to be the fastest way of doing this without using a bunch of unsafe: @@ -39,6 +40,14 @@ impl From for ServerTimestamp { } } +// This lets us use these in hyper header! blocks. +impl FromStr for ServerTimestamp { + type Err = num::ParseFloatError; + fn from_str(s: &str) -> Result { + Ok(ServerTimestamp(f64::from_str(s)?)) + } +} + impl fmt::Display for ServerTimestamp { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { From a73408d0f9f68557db436edf7ff7ddb1835cceb7 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Sat, 28 Apr 2018 01:00:54 -0700 Subject: [PATCH 19/23] Whole mess of tests for batching uploader (it could use more, though) --- sync15-adapter/src/bso_record.rs | 6 + sync15-adapter/src/lib.rs | 17 +- sync15-adapter/src/request.rs | 691 ++++++++++++++++++++++++++++++- sync15-adapter/src/util.rs | 14 +- 4 files changed, 701 insertions(+), 27 deletions(-) diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index 3f21037559..ae93ef9fb4 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -22,6 +22,9 @@ pub struct BsoRecord { pub collection: String, #[serde(skip_serializing)] + // If we don't give it a default, we fail to deserialize + // items we wrote out during tests and such. + #[serde(default = "default_modified")] pub modified: ServerTimestamp, #[serde(skip_serializing_if = "Option::is_none")] @@ -40,6 +43,9 @@ pub struct BsoRecord { pub payload: T, } +// ... ugh. +fn default_modified() -> ServerTimestamp { ServerTimestamp(0.0) } + impl BsoRecord { #[inline] pub fn map_payload(self, mapper: F) -> BsoRecord

where F: FnOnce(T) -> P { diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 4e43f50576..34f00868d5 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -22,6 +22,7 @@ extern crate serde_derive; #[macro_use] extern crate log; +#[macro_use] extern crate serde_json; #[macro_use] @@ -66,7 +67,8 @@ use request::{ XIfUnmodifiedSince, PostResponse, BatchPoster, - PostQueue + PostQueue, + PostResponseHandler, }; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -229,7 +231,7 @@ impl Sync15Service { } } - fn new_post_queue<'a, F: FnMut(PostResponse, bool) -> error::Result<()>>(&'a self, coll: &str, on_response: F) + fn new_post_queue<'a, F: PostResponseHandler>(&'a self, coll: &str, on_response: F) -> error::Result, F>> { let ts = self.last_sync_remote .get(coll) @@ -245,7 +247,13 @@ struct PostWrapper<'a> { } impl<'a> BatchPoster for PostWrapper<'a> { - fn post(&mut self, bytes: &[u8], xius: ServerTimestamp, batch: Option, commit: bool) -> error::Result { + fn post(&self, + bytes: &[u8], + xius: ServerTimestamp, + batch: Option, + commit: bool, + _: &PostQueue) -> error::Result + { let url = CollectionRequest::new(self.coll.clone()) .batch(batch) .commit(commit) @@ -253,6 +261,9 @@ impl<'a> BatchPoster for PostWrapper<'a> { let mut req = self.svc.build_request(Method::Post, url)?; req.headers_mut().set(XIfUnmodifiedSince(xius)); + // It's very annoying that we need to copy the body here, the request + // shouldn't need to take ownership of it... + *req.body_mut() = Some(Vec::from(bytes).into()); let mut resp = self.svc.exec_request(req, false)?; Ok(PostResponse::from_response(&mut resp)?) } diff --git a/sync15-adapter/src/request.rs b/sync15-adapter/src/request.rs index b5128bf62a..711d64f7fd 100644 --- a/sync15-adapter/src/request.rs +++ b/sync15-adapter/src/request.rs @@ -6,8 +6,9 @@ use util::ServerTimestamp; use bso_record::{BsoRecord, EncryptedPayload}; use serde_json; -use std::fmt::{self, Write}; +use std::fmt; use std::collections::HashMap; +use std::default::Default; use url::{Url, UrlQuery, form_urlencoded::Serializer}; use error::{self, Result}; use hyper::{StatusCode}; @@ -193,7 +194,7 @@ impl LimitTracker { assert!(self.can_add_record(record_size), "LimitTracker::record_added caller must check can_add_record"); self.cur_records += 1; - self.cur_bytes += 1; + self.cur_bytes += record_size; } } @@ -228,10 +229,24 @@ pub struct InfoConfiguration { pub max_record_payload_bytes: usize, } -// This is annoying but seems to be the only way to do it. +// This is annoying but seems to be the only way to do it... fn default_max_request_bytes() -> usize { 260 * 1024 } fn default_max_record_payload_bytes() -> usize { 256 * 1024 } +impl Default for InfoConfiguration { + #[inline] + fn default() -> InfoConfiguration { + InfoConfiguration { + max_request_bytes: default_max_request_bytes(), + max_record_payload_bytes: default_max_record_payload_bytes(), + max_post_records: usize::max_value(), + max_post_bytes: usize::max_value(), + max_total_records: usize::max_value(), + max_total_bytes: usize::max_value(), + } + } +} + #[derive(Debug, Clone, Deserialize)] pub struct UploadResult { batch: Option, @@ -281,26 +296,32 @@ pub struct PostQueue { } pub trait BatchPoster { - fn post(&mut self, + /// Note: Last argument (reference to the batch poster) is provided for the purposes of testing + /// Important: Poster should not report non-success HTTP statuses as errors!! + fn post(&self, body: &[u8], - ts: ServerTimestamp, + xius: ServerTimestamp, batch: Option, - commit: bool) -> Result; + commit: bool, + queue: &PostQueue) -> Result; +} + +// We don't just use a FnMut here since we want to override it in mocking for RefCell, +// which we can't do for FnMut since neither FnMut nor RefCell are defined here. Also, this +// is somewhat better for documentation. +pub trait PostResponseHandler { + fn handle_response(&mut self, r: PostResponse, mid_batch: bool) -> Result<()>; } -/// The Poster param takes -/// 1. A slice that will represent the request body -/// 2. XIUS value for the request -/// 3. An optional batch id, which will be None if batching is not supported. -/// 4. A boolean for whether or not this is a batch commit. -/// -/// Note: Poster should not report non-success HTTP statuses as errors!! impl PostQueue where Poster: BatchPoster, - OnResponse: FnMut(PostResponse, bool) -> Result<()> + OnResponse: PostResponseHandler { - pub fn new(config: &InfoConfiguration, ts: ServerTimestamp, poster: Poster, on_response: OnResponse) -> PostQueue { + pub fn new(config: &InfoConfiguration, + ts: ServerTimestamp, + poster: Poster, + on_response: OnResponse) -> PostQueue { PostQueue { poster, on_response, @@ -318,8 +339,8 @@ where fn in_batch(&self) -> bool { match &self.batch { &BatchState::Unsupported | - &BatchState::NoBatch => true, - _ => false + &BatchState::NoBatch => false, + _ => true } } @@ -410,11 +431,15 @@ where let is_commit = want_commit && !batch_id.is_none(); // Weird syntax for calling a function object that is a property. - let resp_or_error = self.poster.post(&self.queued, self.last_modified, batch_id, is_commit); + let resp_or_error = self.poster.post(&self.queued, + self.last_modified, + batch_id, + is_commit, + self); self.queued.truncate(0); - if want_commit { + if want_commit || self.batch == BatchState::Unsupported { self.batch_limits.clear(); } self.post_limits.clear(); @@ -422,7 +447,7 @@ where let resp = resp_or_error?; if !resp.status.is_success() { - (self.on_response)(resp, !want_commit)?; + self.on_response.handle_response(resp, !want_commit)?; bail!(error::unexpected("Expected OnResponse to have bailed out!")); } @@ -430,7 +455,7 @@ where debug!("Committed batch {:?}", self.batch); self.batch = BatchState::NoBatch; self.last_modified = resp.last_modified; - (self.on_response)(resp, false)?; + self.on_response.handle_response(resp, false)?; return Ok(()); } @@ -441,7 +466,8 @@ where } self.last_modified = resp.last_modified; self.batch = BatchState::Unsupported; - (self.on_response)(resp, false)?; + self.batch_limits.clear(); + self.on_response.handle_response(resp, false)?; return Ok(()); } @@ -465,7 +491,7 @@ where self.batch = BatchState::InBatch(batch_id); self.last_modified = resp.last_modified; - (self.on_response)(resp, false)?; + self.on_response.handle_response(resp, true)?; Ok(()) } @@ -474,6 +500,9 @@ where #[cfg(test)] mod test { use super::*; + use std::collections::VecDeque; + use std::cell::RefCell; + use std::rc::Rc; #[test] fn test_url_building() { let base = Url::parse("https://example.com/sync").unwrap(); @@ -500,4 +529,620 @@ mod test { "https://example.com/sync/storage/specific?full=1&limit=10&older=9876.54&newer=1234.56&sort=oldest"); } + + #[derive(Debug, Clone)] + struct PostedData { + body: String, + xius: ServerTimestamp, + batch: Option, + commit: bool, + payload_bytes: usize, + records: usize + } + + impl PostedData { + // Just for convenience when testing + // fn new(body: Body, batch: Batch, ts: Time, commit: bool) -> PostedData + // where Body: Into, + // Batch: Into>, + // S: Into, + // Time: Into + // { + // PostedData { + // body: body.into(), + // xius: ts.into(), + // batch: batch.into().map(|x| x.into()), + // commit + // } + // } + + fn records_as_json(&self) -> Vec { + let values = serde_json::from_str::(&self.body).expect("Posted invalid json"); + // Check that they actually deserialize as what we want + let records_or_err = serde_json::from_value::>>(values.clone()); + records_or_err.expect("Failed to deserialize data"); + serde_json::from_value(values).unwrap() + } + } + + + #[derive(Debug, Clone)] + struct BatchInfo { + id: Option, + posts: Vec, + bytes: usize, + records: usize, + } + + #[derive(Debug, Clone)] + struct TestPoster { + all_posts: Vec, + responses: VecDeque, + batches: Vec, + cur_batch: Option, + cfg: InfoConfiguration, + } + + type TestPosterRef = Rc>; + impl TestPoster { + pub fn new(cfg: &InfoConfiguration, responses: T) -> TestPosterRef + where T: Into> { + Rc::new(RefCell::new(TestPoster { + all_posts: vec![], + responses: responses.into(), + batches: vec![], + cur_batch: None, + cfg: cfg.clone(), + })) + } + // Adds &mut + fn do_post( + &mut self, + body: &[u8], + xius: ServerTimestamp, + batch: Option, + commit: bool, + queue: &PostQueue + ) -> Result { + + let mut post = PostedData { + body: String::from_utf8(body.into()).expect("Posted invalid utf8..."), + batch: batch.clone(), + xius, + commit, + payload_bytes: 0, + records: 0, + }; + + assert!(body.len() <= self.cfg.max_request_bytes); + + let (num_records, record_payload_bytes) = { + let recs = post.records_as_json(); + assert!(recs.len() <= self.cfg.max_post_records); + assert!(recs.len() <= self.cfg.max_total_records); + let payload_bytes: usize = recs.iter().map(|r| { + let len = r["payload"].as_str().expect("Non string payload property").len(); + assert!(len <= self.cfg.max_record_payload_bytes); + len + }).sum(); + assert!(payload_bytes <= self.cfg.max_post_bytes); + assert!(payload_bytes <= self.cfg.max_total_bytes); + + assert_eq!(queue.post_limits.cur_bytes, payload_bytes); + assert_eq!(queue.post_limits.cur_records, recs.len()); + (recs.len(), payload_bytes) + }; + post.payload_bytes = record_payload_bytes; + post.records = num_records; + + self.all_posts.push(post.clone()); + let response = self.responses.pop_front().unwrap(); + + if self.cur_batch.is_none() { + assert!(batch.is_none() || batch == Some("true".into()), + "We shouldn't be in a batch now"); + self.cur_batch = Some(BatchInfo { + id: response.result.batch.clone(), + posts: vec![], + records: 0, + bytes: 0, + }); + } else { + assert_eq!(batch, self.cur_batch.as_ref().unwrap().id, + "We're in a batch but got the wrong batch id"); + } + + { + let batch = self.cur_batch.as_mut().unwrap(); + batch.posts.push(post.clone()); + batch.records += num_records; + batch.bytes += record_payload_bytes; + + assert!(batch.bytes <= self.cfg.max_total_bytes); + assert!(batch.records <= self.cfg.max_total_records); + + assert_eq!(batch.records, queue.batch_limits.cur_records); + assert_eq!(batch.bytes, queue.batch_limits.cur_bytes); + } + + + if commit || response.result.batch.is_none() { + let batch = self.cur_batch.take().unwrap(); + self.batches.push(batch); + } + + Ok(response) + } + + fn do_handle_response(&mut self, _: PostResponse, mid_batch: bool) -> Result<()> { + assert_eq!(mid_batch, self.cur_batch.is_some()); + Ok(()) + } + } + impl BatchPoster for TestPosterRef { + fn post(&self, + body: &[u8], + xius: ServerTimestamp, + batch: Option, + commit: bool, + queue: &PostQueue) -> Result { + self.borrow_mut().do_post(body, xius, batch, commit, queue) + } + } + + impl PostResponseHandler for TestPosterRef { + fn handle_response(&mut self, r: PostResponse, mid_batch: bool) -> Result<()> { + self.borrow_mut().do_handle_response(r, mid_batch) + } + } + + type MockedPostQueue = PostQueue; + + fn pq_test_setup(cfg: InfoConfiguration, lm: f64, resps: Vec) -> (MockedPostQueue, TestPosterRef) { + let tester = TestPoster::new(&cfg, resps); + let pq = PostQueue::new(&cfg, ServerTimestamp(lm), tester.clone(), tester.clone()); + (pq, tester) + } + + fn fake_response<'a, T: Into>>(status: StatusCode, lm: f64, batch: T) -> PostResponse { + PostResponse { + status, + last_modified: ServerTimestamp(lm), + result: UploadResult { + batch: batch.into().map(|x| x.into()), + failed: HashMap::new(), + success: vec![], + } + } + } + + lazy_static! { + // ~40b + static ref PAYLOAD_OVERHEAD: usize = { + let payload = EncryptedPayload { + iv: "".into(), + hmac: "".into(), + ciphertext: "".into() + }; + serde_json::to_string(&payload).unwrap().len() + }; + // ~80b + static ref TOTAL_RECORD_OVERHEAD: usize = { + let val = serde_json::to_value(BsoRecord { + id: "".into(), + collection: "".into(), + modified: ServerTimestamp(0.0), + sortindex: None, + ttl: None, + payload: EncryptedPayload { + iv: "".into(), + hmac: "".into(), + ciphertext: "".into() + }, + }).unwrap(); + serde_json::to_string(&val).unwrap().len() + }; + // There's some subtlety in how we calulate this having to do with the fact that + // the quotes in the payload are escaped but the escape chars count to the request len + // and *not* to the payload len (the payload len check happens after json parsing the + // top level object). + static ref NON_PAYLOAD_OVERHEAD: usize = { + *TOTAL_RECORD_OVERHEAD - *PAYLOAD_OVERHEAD + }; + } + + // Actual record size (for max_request_len) will be larger by some amount + fn make_record(payload_size: usize) -> BsoRecord { + assert!(payload_size > *PAYLOAD_OVERHEAD); + let ciphertext_len = payload_size - *PAYLOAD_OVERHEAD; + BsoRecord { + id: "".into(), + collection: "".into(), + modified: ServerTimestamp(0.0), + sortindex: None, + ttl: None, + payload: EncryptedPayload { + iv: "".into(), + hmac: "".into(), + ciphertext: "x".repeat(ciphertext_len) + } + } + } + + fn request_bytes_for_payloads(payloads: &[usize]) -> usize { + 1 + payloads.iter().map(|&size| size + 1 + *NON_PAYLOAD_OVERHEAD).sum::() + } + + #[test] + fn test_pq_basic() { + let cfg = InfoConfiguration { + max_request_bytes: 1000, + max_record_payload_bytes: 1000, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Ok, time + 100.0, None), + ]); + + pq.enqueue(&make_record(100)).unwrap(); + pq.flush(true).unwrap(); + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 1); + assert_eq!(t.batches.len(), 1); + assert_eq!(t.batches[0].posts.len(), 1); + assert_eq!(t.batches[0].records, 1); + assert_eq!(t.batches[0].bytes, 100); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[100])); + } + + #[test] + fn test_pq_max_request_bytes_no_batch() { + let cfg = InfoConfiguration { + max_request_bytes: 250, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Ok, time + 100.0, None), + fake_response(StatusCode::Ok, time + 200.0, None), + ]); + + // Note that the total record overhead is around 85 bytes + let payload_size = 100 - *NON_PAYLOAD_OVERHEAD; + pq.enqueue(&make_record(payload_size)).unwrap(); // total size == 102; [r] + pq.enqueue(&make_record(payload_size)).unwrap(); // total size == 203; [r,r] + pq.enqueue(&make_record(payload_size)).unwrap(); // too big, 2nd post. + pq.flush(true).unwrap(); + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 2); + assert_eq!(t.batches.len(), 2); + assert_eq!(t.batches[0].posts.len(), 1); + assert_eq!(t.batches[0].records, 2); + assert_eq!(t.batches[0].bytes, payload_size * 2); + assert_eq!(t.batches[0].posts[0].batch, Some("true".into())); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[payload_size, payload_size])); + + assert_eq!(t.batches[1].posts.len(), 1); + assert_eq!(t.batches[1].records, 1); + assert_eq!(t.batches[1].bytes, payload_size); + // We know at this point that the server does not support batching. + assert_eq!(t.batches[1].posts[0].batch, None); + assert_eq!(t.batches[1].posts[0].commit, false); + assert_eq!(t.batches[1].posts[0].body.len(), + request_bytes_for_payloads(&[payload_size])); + } + + #[test] + fn test_pq_max_record_payload_bytes_no_batch() { + let cfg = InfoConfiguration { + max_record_payload_bytes: 150, + max_request_bytes: 350, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Ok, time + 100.0, None), + fake_response(StatusCode::Ok, time + 200.0, None), + ]); + + // Note that the total record overhead is around 85 bytes + let payload_size = 100 - *NON_PAYLOAD_OVERHEAD; + pq.enqueue(&make_record(payload_size)).unwrap(); // total size == 102; [r] + let enqueued = pq.enqueue(&make_record(151)).unwrap(); // still 102 + assert!(!enqueued, "Should not have fit"); + pq.enqueue(&make_record(payload_size)).unwrap(); + pq.flush(true).unwrap(); + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 1); + assert_eq!(t.batches.len(), 1); + assert_eq!(t.batches[0].posts.len(), 1); + assert_eq!(t.batches[0].records, 2); + assert_eq!(t.batches[0].bytes, payload_size * 2); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[payload_size, payload_size])); + } + + #[test] + fn test_pq_single_batch() { + let cfg = InfoConfiguration::default(); + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Accepted, time, Some("1234")), + ]); + + let payload_size = 100 - *NON_PAYLOAD_OVERHEAD; + pq.enqueue(&make_record(payload_size)).unwrap(); + pq.enqueue(&make_record(payload_size)).unwrap(); + pq.enqueue(&make_record(payload_size)).unwrap(); + pq.flush(true).unwrap(); + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 1); + assert_eq!(t.batches.len(), 1); + assert_eq!(t.batches[0].id.as_ref().unwrap(), "1234"); + assert_eq!(t.batches[0].posts.len(), 1); + assert_eq!(t.batches[0].records, 3); + assert_eq!(t.batches[0].bytes, payload_size * 3); + assert_eq!(t.batches[0].posts[0].commit, true); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[payload_size, payload_size, payload_size])); + } + + #[test] + fn test_pq_multi_post_batch_bytes() { + let cfg = InfoConfiguration { + max_post_bytes: 200, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("1234")), + ]); + + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.flush(true).unwrap(); // COMMIT + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 2); + assert_eq!(t.batches.len(), 1); + assert_eq!(t.batches[0].posts.len(), 2); + assert_eq!(t.batches[0].records, 3); + assert_eq!(t.batches[0].bytes, 300); + + assert_eq!(t.batches[0].posts[0].batch.as_ref().unwrap(), "true"); + assert_eq!(t.batches[0].posts[0].records, 2); + assert_eq!(t.batches[0].posts[0].payload_bytes, 200); + assert_eq!(t.batches[0].posts[0].commit, false); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[100, 100])); + + assert_eq!(t.batches[0].posts[1].batch.as_ref().unwrap(), "1234"); + assert_eq!(t.batches[0].posts[1].records, 1); + assert_eq!(t.batches[0].posts[1].payload_bytes, 100); + assert_eq!(t.batches[0].posts[1].commit, true); + assert_eq!(t.batches[0].posts[1].body.len(), + request_bytes_for_payloads(&[100])); + } + + + #[test] + fn test_pq_multi_post_batch_records() { + let cfg = InfoConfiguration { + max_post_records: 3, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("1234")), + ]); + + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.flush(true).unwrap(); // COMMIT + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 3); + assert_eq!(t.batches.len(), 1); + assert_eq!(t.batches[0].posts.len(), 3); + assert_eq!(t.batches[0].records, 7); + assert_eq!(t.batches[0].bytes, 700); + + assert_eq!(t.batches[0].posts[0].batch.as_ref().unwrap(), "true"); + assert_eq!(t.batches[0].posts[0].records, 3); + assert_eq!(t.batches[0].posts[0].payload_bytes, 300); + assert_eq!(t.batches[0].posts[0].commit, false); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[100, 100, 100])); + + assert_eq!(t.batches[0].posts[1].batch.as_ref().unwrap(), "1234"); + assert_eq!(t.batches[0].posts[1].records, 3); + assert_eq!(t.batches[0].posts[1].payload_bytes, 300); + assert_eq!(t.batches[0].posts[1].commit, false); + assert_eq!(t.batches[0].posts[1].body.len(), + request_bytes_for_payloads(&[100, 100, 100])); + + assert_eq!(t.batches[0].posts[2].batch.as_ref().unwrap(), "1234"); + assert_eq!(t.batches[0].posts[2].records, 1); + assert_eq!(t.batches[0].posts[2].payload_bytes, 100); + assert_eq!(t.batches[0].posts[2].commit, true); + assert_eq!(t.batches[0].posts[2].body.len(), + request_bytes_for_payloads(&[100])); + } + + #[test] + fn test_pq_multi_post_multi_batch_records() { + let cfg = InfoConfiguration { + max_post_records: 3, + max_total_records: 5, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("abcd")), + fake_response(StatusCode::Accepted, time, Some("abcd")), + ]); + + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + COMMIT + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.flush(true).unwrap(); // COMMIT + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 4); + assert_eq!(t.batches.len(), 2); + assert_eq!(t.batches[0].posts.len(), 2); + assert_eq!(t.batches[1].posts.len(), 2); + + assert_eq!(t.batches[0].records, 5); + assert_eq!(t.batches[1].records, 4); + + assert_eq!(t.batches[0].bytes, 500); + assert_eq!(t.batches[1].bytes, 400); + + assert_eq!(t.batches[0].posts[0].batch.as_ref().unwrap(), "true"); + assert_eq!(t.batches[0].posts[0].records, 3); + assert_eq!(t.batches[0].posts[0].payload_bytes, 300); + assert_eq!(t.batches[0].posts[0].commit, false); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[100, 100, 100])); + + assert_eq!(t.batches[0].posts[1].batch.as_ref().unwrap(), "1234"); + assert_eq!(t.batches[0].posts[1].records, 2); + assert_eq!(t.batches[0].posts[1].payload_bytes, 200); + assert_eq!(t.batches[0].posts[1].commit, true); + assert_eq!(t.batches[0].posts[1].body.len(), + request_bytes_for_payloads(&[100, 100])); + + + assert_eq!(t.batches[1].posts[0].batch.as_ref().unwrap(), "true"); + assert_eq!(t.batches[1].posts[0].records, 3); + assert_eq!(t.batches[1].posts[0].payload_bytes, 300); + assert_eq!(t.batches[1].posts[0].commit, false); + assert_eq!(t.batches[1].posts[0].body.len(), + request_bytes_for_payloads(&[100, 100, 100])); + + assert_eq!(t.batches[1].posts[1].batch.as_ref().unwrap(), "abcd"); + assert_eq!(t.batches[1].posts[1].records, 1); + assert_eq!(t.batches[1].posts[1].payload_bytes, 100); + assert_eq!(t.batches[1].posts[1].commit, true); + assert_eq!(t.batches[1].posts[1].body.len(), + request_bytes_for_payloads(&[100])); + } + + #[test] + fn test_pq_multi_post_multi_batch_bytes() { + let cfg = InfoConfiguration { + max_post_bytes: 300, + max_total_bytes: 500, + ..InfoConfiguration::default() + }; + let time = 11111111.0; + let (mut pq, tester) = pq_test_setup(cfg, time, vec![ + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("1234")), + fake_response(StatusCode::Accepted, time, Some("abcd")), + fake_response(StatusCode::Accepted, time, Some("abcd")), + ]); + + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + COMMIT + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + pq.enqueue(&make_record(100)).unwrap(); + // POST + pq.enqueue(&make_record(100)).unwrap(); + pq.flush(true).unwrap(); // COMMIT + + let t = tester.borrow(); + assert!(t.cur_batch.is_none()); + assert_eq!(t.all_posts.len(), 4); + assert_eq!(t.batches.len(), 2); + assert_eq!(t.batches[0].posts.len(), 2); + assert_eq!(t.batches[1].posts.len(), 2); + + assert_eq!(t.batches[0].records, 5); + assert_eq!(t.batches[1].records, 4); + + assert_eq!(t.batches[0].bytes, 500); + assert_eq!(t.batches[1].bytes, 400); + + assert_eq!(t.batches[0].posts[0].batch.as_ref().unwrap(), "true"); + assert_eq!(t.batches[0].posts[0].records, 3); + assert_eq!(t.batches[0].posts[0].payload_bytes, 300); + assert_eq!(t.batches[0].posts[0].commit, false); + assert_eq!(t.batches[0].posts[0].body.len(), + request_bytes_for_payloads(&[100, 100, 100])); + + assert_eq!(t.batches[0].posts[1].batch.as_ref().unwrap(), "1234"); + assert_eq!(t.batches[0].posts[1].records, 2); + assert_eq!(t.batches[0].posts[1].payload_bytes, 200); + assert_eq!(t.batches[0].posts[1].commit, true); + assert_eq!(t.batches[0].posts[1].body.len(), + request_bytes_for_payloads(&[100, 100])); + + + assert_eq!(t.batches[1].posts[0].batch.as_ref().unwrap(), "true"); + assert_eq!(t.batches[1].posts[0].records, 3); + assert_eq!(t.batches[1].posts[0].payload_bytes, 300); + assert_eq!(t.batches[1].posts[0].commit, false); + assert_eq!(t.batches[1].posts[0].body.len(), + request_bytes_for_payloads(&[100, 100, 100])); + + assert_eq!(t.batches[1].posts[1].batch.as_ref().unwrap(), "abcd"); + assert_eq!(t.batches[1].posts[1].records, 1); + assert_eq!(t.batches[1].posts[1].payload_bytes, 100); + assert_eq!(t.batches[1].posts[1].commit, true); + assert_eq!(t.batches[1].posts[1].body.len(), + request_bytes_for_payloads(&[100])); + } + + // TODO: Test + // + // - error cases!!! We don't test our handling of server errors at all! + // - mixed bytes/record limits + // + // A lot of these have good examples in test_postqueue.js on deskftop sync + } diff --git a/sync15-adapter/src/util.rs b/sync15-adapter/src/util.rs index 0930f2471e..d00fcefc88 100644 --- a/sync15-adapter/src/util.rs +++ b/sync15-adapter/src/util.rs @@ -24,6 +24,12 @@ pub fn base16_encode(bytes: &[u8]) -> String { /// Typesafe way to manage server timestamps without accidentally mixing them up with /// local ones. +/// +/// TODO: We should probably store this as milliseconds (or something) for stability and to get +/// Eq/Ord. The server guarantees that these are formatted to the hundreds place (not sure if this +/// is documented but the code does it intentionally...). This would also let us throw out negative +/// and NaN timestamps, which the server certainly won't send, but the guarantee would make me feel +/// better. #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Deserialize, Serialize)] pub struct ServerTimestamp(pub f64); @@ -59,7 +65,7 @@ pub const SERVER_EPOCH: ServerTimestamp = ServerTimestamp(0.0); impl ServerTimestamp { /// Returns None if `other` is later than `self` (Duration may not represent - /// negative timespans in rust) + /// negative timespans in rust). #[inline] pub fn duration_since(self, other: ServerTimestamp) -> Option { let delta = self.0 - other.0; @@ -73,6 +79,12 @@ impl ServerTimestamp { Some(Duration::new(secs as u64, nanos)) } } + + /// Get the milliseconds for the timestamp. + #[inline] + pub fn as_millis(self) -> u64 { + (self.0 * 1000.0).floor() as u64 + } } #[cfg(test)] From fa977ef3fa5ffa9b4ba04b9f4c35e12a89b3d3c3 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 30 Apr 2018 14:17:46 -0700 Subject: [PATCH 20/23] Get batch upload demonstratably working --- sync15-adapter/examples/boxlocker-parity.rs | 95 ++++++++++++++++++++- sync15-adapter/src/bso_record.rs | 65 ++++++++++++-- sync15-adapter/src/error.rs | 11 +++ sync15-adapter/src/lib.rs | 93 ++++++++++++++++++-- sync15-adapter/src/record_types.rs | 7 ++ sync15-adapter/src/request.rs | 89 +++++++++++++++---- sync15-adapter/src/util.rs | 23 ++++- 7 files changed, 348 insertions(+), 35 deletions(-) diff --git a/sync15-adapter/examples/boxlocker-parity.rs b/sync15-adapter/examples/boxlocker-parity.rs index b0e0b015f9..f7688f1c85 100644 --- a/sync15-adapter/examples/boxlocker-parity.rs +++ b/sync15-adapter/examples/boxlocker-parity.rs @@ -12,10 +12,11 @@ extern crate serde_json; extern crate env_logger; -use std::io::Read; +use std::io::{self, Read, Write}; use std::error::Error; use std::fs; use std::process; +use std::time; use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; use sync15_adapter as sync; @@ -61,6 +62,60 @@ fn do_auth(recur: bool) -> Result> { } } +fn prompt_string>(prompt: S) -> Option { + print!("{}: ", prompt.as_ref()); + let _ = io::stdout().flush(); // Don't care if flush fails really. + let mut s = String::new(); + io::stdin().read_line(&mut s).expect("Failed to read line..."); + if let Some('\n') = s.chars().next_back() { s.pop(); } + if let Some('\r') = s.chars().next_back() { s.pop(); } + if s.len() == 0 { + None + } else { + Some(s) + } +} + +fn read_login() -> sync::record_types::PasswordRecord { + let username = prompt_string("username").unwrap_or(String::new()); + let password = prompt_string("password").unwrap_or(String::new()); + let form_submit_url = prompt_string("form_submit_url"); + let hostname = prompt_string("hostname"); + let http_realm = prompt_string("http_realm"); + let username_field = prompt_string("username_field").unwrap_or(String::new()); + let password_field = prompt_string("password_field").unwrap_or(String::new()); + let since_unix_epoch = time::SystemTime::now().duration_since(time::UNIX_EPOCH).unwrap(); + let dur_ms = since_unix_epoch.as_secs() * 1000 + ((since_unix_epoch.subsec_nanos() / 1_000_000) as u64); + let ms_i64 = dur_ms as i64; + sync::record_types::PasswordRecord { + id: sync::util::random_guid().unwrap(), + username, + password, + username_field, + password_field, + form_submit_url, + http_realm, + hostname, + time_created: ms_i64, + time_password_changed: ms_i64, + times_used: None, + time_last_used: Some(ms_i64), + } +} + +fn prompt_bool(msg: &str) -> Option { + let result = prompt_string(msg); + result.and_then(|r| match r.chars().next().unwrap() { + 'y' | 'Y' | 't' | 'T' => Some(true), + 'n' | 'N' | 'f' | 'F' => Some(false), + _ => None + }) +} + +fn prompt_chars(msg: &str) -> Option { + prompt_string(msg).and_then(|r| r.chars().next()) +} + fn start() -> Result<(), Box> { let oauth_data = do_auth(false)?; @@ -84,6 +139,44 @@ fn start() -> Result<(), Box> { println!("{:?}", pw.payload); } + if !prompt_bool("Would you like to make changes? [y/N]").unwrap_or(false) { + return Ok(()); + } + + let mut ids: Vec = passwords.iter().map(|p| p.id.clone()).collect(); + + let mut upd = sync::CollectionUpdate::new(&svc, false); + loop { + match prompt_chars("Add, delete, or commit [adc]:").unwrap_or('s') { + 'A' | 'a' => { + let record = read_login(); + upd.add_record(record); + }, + 'D' | 'd' => { + for (i, id) in ids.iter().enumerate() { + println!("{}: {}", i, id); + } + if let Some(index) = prompt_string("Index to delete (enter index)").and_then(|x| x.parse::().ok()) { + let result = ids.swap_remove(index); + upd.add_tombstone(result); + } else { + println!("???"); + } + }, + 'C' | 'c' => { + println!("committing!"); + let (good, bad) = upd.upload()?; + println!("Uploded {} ids successfully, and {} unsuccessfully", + good.len(), bad.len()); + break; + }, + c => { + println!("Unknown action '{}', exiting.", c); + break; + } + } + } + Ok(()) } diff --git a/sync15-adapter/src/bso_record.rs b/sync15-adapter/src/bso_record.rs index ae93ef9fb4..73c25f5db6 100644 --- a/sync15-adapter/src/bso_record.rs +++ b/sync15-adapter/src/bso_record.rs @@ -24,7 +24,7 @@ pub struct BsoRecord { #[serde(skip_serializing)] // If we don't give it a default, we fail to deserialize // items we wrote out during tests and such. - #[serde(default = "default_modified")] + #[serde(default = "ServerTimestamp::default")] pub modified: ServerTimestamp, #[serde(skip_serializing_if = "Option::is_none")] @@ -43,9 +43,6 @@ pub struct BsoRecord { pub payload: T, } -// ... ugh. -fn default_modified() -> ServerTimestamp { ServerTimestamp(0.0) } - impl BsoRecord { #[inline] pub fn map_payload(self, mapper: F) -> BsoRecord

where F: FnOnce(T) -> P { @@ -70,6 +67,13 @@ impl BsoRecord { pub trait Sync15Record: Clone + DeserializeOwned + Serialize { fn collection_tag() -> &'static str; fn record_id(&self) -> &str; + + // Max TTL is actually 31536000, weirdly. + #[inline] + fn ttl() -> Option { None } + + #[inline] + fn sortindex(&self) -> Option { None } } impl From for BsoRecord where T: Sync15Record { @@ -77,11 +81,11 @@ impl From for BsoRecord where T: Sync15Record { fn from(payload: T) -> BsoRecord { let id = payload.record_id().into(); let collection = T::collection_tag().into(); + let sortindex = payload.sortindex(); BsoRecord { - id, collection, payload, + id, collection, payload, sortindex, modified: ServerTimestamp(0.0), - sortindex: None, - ttl: None, + ttl: T::ttl(), } } } @@ -113,6 +117,21 @@ impl DerefMut for BsoRecord { } } +impl BsoRecord { + /// If T is a Sync15Record, then you can/should just use record.into() instead! + #[inline] + pub fn new_non_record, C: Into>(id: I, coll: C, payload: T) -> BsoRecord { + BsoRecord { + id: id.into(), + collection: coll.into(), + ttl: None, + sortindex: None, + modified: ServerTimestamp::default(), + payload, + } + } +} + // Contains the methods to automatically deserialize the payload to/from json. mod as_json { use serde_json; @@ -233,4 +252,36 @@ mod tests { record.payload.serialized_len()) } + #[derive(Debug, Clone, Serialize, Deserialize)] + struct MyRecordType { + id: String, + data: String, + idx: i32, + } + + impl Sync15Record for MyRecordType { + fn collection_tag() -> &'static str { "my_cool_records" } + fn record_id(&self) -> &str { &self.id } + // 3 years in seconds + fn ttl() -> Option { Some(3 * 365 * 24 * 60 * 60) } + fn sortindex(&self) -> Option { Some(self.idx) } + } + + #[test] + fn test_sync15record() { + let record: MyRecordType = MyRecordType { + id: "aaabbbcccddd".into(), + data: "this is extremely good and cool data".into(), + idx: 9001 + }; + let bso: BsoRecord = record.into(); + let s = serde_json::to_string(&bso).unwrap(); + let out: serde_json::Value = serde_json::from_str(&s).unwrap(); + let ttl = 3*365*24*60*60; + assert_eq!(out["ttl"], json!(ttl)); + assert_eq!(out["sortindex"], json!(9001)); + assert_eq!(out["id"], json!("aaabbbcccddd")); + assert_eq!(out["collection"], json!("my_cool_records")); + } + } diff --git a/sync15-adapter/src/error.rs b/sync15-adapter/src/error.rs index 55e1f591c2..61509d0744 100644 --- a/sync15-adapter/src/error.rs +++ b/sync15-adapter/src/error.rs @@ -67,6 +67,17 @@ error_chain! { description("Record is larger than the maximum size allowed by the server") display("Record is larger than the maximum size allowed by the server") } + + BatchInterrupted { + description("Batch interrupted: server responded with 412") + display("Batch interrupted: server responded with 412") + } + + RecordUploadFailed(problems: ::std::collections::HashMap) { + description("Some records failed to upload, but success was required for the collection") + display("Several records failed to upload ({}), but success was required for the collection", + problems.len()) + } } } diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 34f00868d5..874531360d 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -46,7 +46,7 @@ use util::ServerTimestamp; use std::cell::Cell; use std::time::{Duration}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use key_bundle::KeyBundle; use reqwest::{ @@ -69,6 +69,7 @@ use request::{ BatchPoster, PostQueue, PostResponseHandler, + NormalResponseHandler, }; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -93,9 +94,6 @@ pub struct Sync15Service { last_sync_remote: HashMap, } - - - impl Sync15Service { pub fn new(init_params: Sync15ServiceInit) -> error::Result { let root_key = KeyBundle::from_ksync_base64(&init_params.sync_key)?; @@ -231,13 +229,19 @@ impl Sync15Service { } } - fn new_post_queue<'a, F: PostResponseHandler>(&'a self, coll: &str, on_response: F) + pub fn last_modified(&self, coll: &str) -> Option { + self.last_sync_remote.get(coll).cloned() + } + + pub fn last_modified_or_zero(&self, coll: &str) -> ServerTimestamp { + self.last_modified(coll).unwrap_or(util::SERVER_EPOCH) + } + + fn new_post_queue<'a, F: PostResponseHandler>(&'a self, coll: &str, lm: Option, on_response: F) -> error::Result, F>> { - let ts = self.last_sync_remote - .get(coll) - .ok_or_else(|| error::unexpected(format!("Unknown collection {}", coll)))?; + let ts = lm.unwrap_or_else(|| self.last_modified_or_zero(&coll)); let pw = PostWrapper { svc: self, coll: coll.into() }; - Ok(PostQueue::new(self.server_config.as_ref().unwrap(), *ts, pw, on_response)) + Ok(PostQueue::new(self.server_config.as_ref().unwrap(), ts, pw, on_response)) } } @@ -260,6 +264,7 @@ impl<'a> BatchPoster for PostWrapper<'a> { .build_url(Url::parse(&self.svc.tsc.token().api_endpoint)?)?; let mut req = self.svc.build_request(Method::Post, url)?; + req.headers_mut().set(hyper::header::ContentType::json()); req.headers_mut().set(XIfUnmodifiedSince(xius)); // It's very annoying that we need to copy the body here, the request // shouldn't need to take ownership of it... @@ -268,3 +273,73 @@ impl<'a> BatchPoster for PostWrapper<'a> { Ok(PostResponse::from_response(&mut resp)?) } } + +#[derive(Debug, Clone)] +pub struct CollectionUpdate<'a, T> { + svc: &'a Sync15Service, + last_sync: ServerTimestamp, + to_update: Vec>, + allow_dropped_records: bool, + queued_ids: HashSet +} + +impl<'a, T> CollectionUpdate<'a, T> where T: Sync15Record { + pub fn new(svc: &'a Sync15Service, allow_dropped_records: bool) -> CollectionUpdate<'a, T> { + let coll = T::collection_tag(); + let ts = svc.last_modified_or_zero(coll); + CollectionUpdate { + svc, + last_sync: ts, + to_update: vec![], + allow_dropped_records, + queued_ids: HashSet::new(), + } + } + + pub fn add(&mut self, rec_or_tombstone: MaybeTombstone) { + // Block to limit scope of the `id` borrow. + { + let id = rec_or_tombstone.record_id(); + // Should this be an Err and not an assertion? + assert!(!self.queued_ids.contains(id), + "Attempt to update ID multiple times in the same batch {}", id); + self.queued_ids.insert(id.into()); + } + self.to_update.push(rec_or_tombstone); + } + + pub fn add_record(&mut self, record: T) { + self.add(NonTombstone(record)); + } + + pub fn add_tombstone(&mut self, id: String) { + self.add(MaybeTombstone::tombstone(id)); + } + + /// Returns a list of the IDs that failed if allowed_dropped_records is true, otherwise + /// returns an empty vec. + pub fn upload(self) -> error::Result<(Vec, Vec)> { + let mut failed = vec![]; + let key = self.svc.key_for_collection(T::collection_tag())?; + let mut q = self.svc.new_post_queue(T::collection_tag(), Some(self.last_sync), + NormalResponseHandler::new(self.allow_dropped_records))?; + + for record in self.to_update.into_iter() { + let record_cleartext: BsoRecord> = record.into(); + let encrypted = record_cleartext.encrypt(key)?; + let enqueued = q.enqueue(&encrypted)?; + if !enqueued && !self.allow_dropped_records { + bail!(error::ErrorKind::RecordTooLargeError); + } + } + + q.flush(true)?; + let (successful_ids, mut failed_ids) = q.successful_and_failed_ids(); + failed_ids.append(&mut failed); + if !self.allow_dropped_records { + assert_eq!(failed_ids.len(), 0, + "Bug: Should have failed by now if we aren't allowing dropped records"); + } + Ok((successful_ids, failed_ids)) + } +} diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 9cd4f98d1b..872c301411 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -63,12 +63,19 @@ impl MaybeTombstone { impl Sync15Record for MaybeTombstone where T: Sync15Record { fn collection_tag() -> &'static str { T::collection_tag() } + fn ttl() -> Option { T::ttl() } fn record_id(&self) -> &str { match self { &Tombstone { ref id, .. } => id, &NonTombstone(ref record) => record.record_id() } } + fn sortindex(&self) -> Option { + match self { + &Tombstone { .. } => None, + &NonTombstone(ref record) => record.sortindex() + } + } } impl BsoRecord> { diff --git a/sync15-adapter/src/request.rs b/sync15-adapter/src/request.rs index 711d64f7fd..da569af328 100644 --- a/sync15-adapter/src/request.rs +++ b/sync15-adapter/src/request.rs @@ -14,7 +14,6 @@ use error::{self, Result}; use hyper::{StatusCode}; use reqwest::Response; - #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum RequestOrder { Oldest, Newest, Index } @@ -250,9 +249,11 @@ impl Default for InfoConfiguration { #[derive(Debug, Clone, Deserialize)] pub struct UploadResult { batch: Option, - /// Maps record id => why failde + /// Maps record id => why failed + #[serde(default = "HashMap::new")] pub failed: HashMap, /// Vec of ids + #[serde(default = "Vec::new")] pub success: Vec } @@ -267,6 +268,7 @@ pub struct PostResponse { impl PostResponse { pub fn from_response(r: &mut Response) -> Result { let result: UploadResult = r.json()?; + // TODO Can this happen in error cases? let last_modified = r.headers().get::().map(|h| **h).ok_or_else(|| error::unexpected("Server didn't send X-Last-Modified header"))?; let status = r.status(); @@ -285,6 +287,7 @@ pub enum BatchState { #[derive(Debug)] pub struct PostQueue { poster: Post, + // XXX: It's cludgey that this needs to be pub just so we can get at the data it saves... on_response: OnResponse, post_limits: LimitTracker, batch_limits: LimitTracker, @@ -313,6 +316,56 @@ pub trait PostResponseHandler { fn handle_response(&mut self, r: PostResponse, mid_batch: bool) -> Result<()>; } + +#[derive(Debug, Clone)] +pub(crate) struct NormalResponseHandler { + pub failed_ids: Vec, + pub successful_ids: Vec, + pub allow_failed: bool, + pub pending_failed: Vec, + pub pending_success: Vec, +} + +impl NormalResponseHandler { + pub fn new(allow_failed: bool) -> NormalResponseHandler { + NormalResponseHandler { + failed_ids: vec![], + successful_ids: vec![], + pending_failed: vec![], + pending_success: vec![], + allow_failed, + } + } +} + +impl PostResponseHandler for NormalResponseHandler { + fn handle_response(&mut self, r: PostResponse, mid_batch: bool) -> error::Result<()> { + if !r.status.is_success() { + warn!("Got failure status from server while posting: {}", r.status); + if r.status == StatusCode::PreconditionFailed { + bail!(error::ErrorKind::BatchInterrupted); + } else { + bail!(error::ErrorKind::StorageHttpError(r.status, + "collection storage (TODO: record route somewhere)".into())); + } + } + if r.result.failed.len() > 0 && !self.allow_failed { + bail!(error::ErrorKind::RecordUploadFailed(r.result.failed.clone())); + } + for id in r.result.success.iter() { + self.pending_success.push(id.clone()); + } + for kv in r.result.failed.iter() { + self.pending_failed.push(kv.0.clone()); + } + if !mid_batch { + self.successful_ids.append(&mut self.pending_success); + self.failed_ids.append(&mut self.pending_failed); + } + Ok(()) + } +} + impl PostQueue where Poster: BatchPoster, @@ -497,6 +550,23 @@ where } } +impl PostQueue { + pub(crate) fn successful_and_failed_ids(&mut self) -> (Vec, Vec) { + let mut good = Vec::with_capacity(self.on_response.successful_ids.len()); + // includes pending_success since they weren't committed! + let mut bad = Vec::with_capacity(self.on_response.failed_ids.len() + + self.on_response.pending_failed.len() + + self.on_response.pending_success.len()); + good.append(&mut self.on_response.successful_ids); + + bad.append(&mut self.on_response.failed_ids); + bad.append(&mut self.on_response.pending_failed); + bad.append(&mut self.on_response.pending_success); + + (good, bad) + } +} + #[cfg(test)] mod test { use super::*; @@ -541,21 +611,6 @@ mod test { } impl PostedData { - // Just for convenience when testing - // fn new(body: Body, batch: Batch, ts: Time, commit: bool) -> PostedData - // where Body: Into, - // Batch: Into>, - // S: Into, - // Time: Into - // { - // PostedData { - // body: body.into(), - // xius: ts.into(), - // batch: batch.into().map(|x| x.into()), - // commit - // } - // } - fn records_as_json(&self) -> Vec { let values = serde_json::from_str::(&self.body).expect("Posted invalid json"); // Check that they actually deserialize as what we want diff --git a/sync15-adapter/src/util.rs b/sync15-adapter/src/util.rs index d00fcefc88..27dbc6a67f 100644 --- a/sync15-adapter/src/util.rs +++ b/sync15-adapter/src/util.rs @@ -6,6 +6,8 @@ use std::convert::From; use std::time::Duration; use std::{fmt, num}; use std::str::FromStr; +use openssl; +use base64; pub fn base16_encode(bytes: &[u8]) -> String { // This seems to be the fastest way of doing this without using a bunch of unsafe: @@ -22,6 +24,12 @@ pub fn base16_encode(bytes: &[u8]) -> String { String::from_utf8(result).unwrap() } +pub fn random_guid() -> Result { + let mut bytes = vec![0u8; 9]; + openssl::rand::rand_bytes(&mut bytes)?; + Ok(base64::encode_config(&bytes, base64::URL_SAFE_NO_PAD)) +} + /// Typesafe way to manage server timestamps without accidentally mixing them up with /// local ones. /// @@ -30,7 +38,7 @@ pub fn base16_encode(bytes: &[u8]) -> String { /// is documented but the code does it intentionally...). This would also let us throw out negative /// and NaN timestamps, which the server certainly won't send, but the guarantee would make me feel /// better. -#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Deserialize, Serialize)] +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Deserialize, Serialize, Default)] pub struct ServerTimestamp(pub f64); impl From for f64 { @@ -90,6 +98,8 @@ impl ServerTimestamp { #[cfg(test)] mod test { use super::*; + use std::collections::HashSet; + #[test] fn test_server_timestamp() { let t0 = ServerTimestamp(10300.15); @@ -111,4 +121,15 @@ mod test { assert_eq!(base16_encode(&[0x00, 0x01, 0x02, 0x03, 0x0a]), "000102030a"); assert_eq!(base16_encode(&[0x00, 0x10, 0x20, 0x30, 0xa0]), "00102030a0"); } + + #[test] + fn test_gen_guid() { + let mut set = HashSet::new(); + for _ in 0..100 { + let res = random_guid().unwrap(); + assert_eq!(res.len(), 12); + assert!(!set.contains(&res)); + set.insert(res); + } + } } From 635663999a1967d8f3736df662d657a5f9a93d0f Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Tue, 1 May 2018 17:12:48 -0700 Subject: [PATCH 21/23] Implement an FFI for sync15-adapter --- Cargo.toml | 3 +- sync15-adapter/Cargo.toml | 4 +- sync15-adapter/examples/boxlocker-parity.rs | 5 +- sync15-adapter/ffi/Cargo.toml | 14 ++ sync15-adapter/ffi/README.md | 9 + sync15-adapter/ffi/src/lib.rs | 207 ++++++++++++++++++++ sync15-adapter/ffi/sync_adapter.h | 63 ++++++ sync15-adapter/src/lib.rs | 14 +- sync15-adapter/src/token.rs | 3 +- 9 files changed, 308 insertions(+), 14 deletions(-) create mode 100644 sync15-adapter/ffi/Cargo.toml create mode 100644 sync15-adapter/ffi/README.md create mode 100644 sync15-adapter/ffi/src/lib.rs create mode 100644 sync15-adapter/ffi/sync_adapter.h diff --git a/Cargo.toml b/Cargo.toml index 361b6a7815..44051a1ff1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ "boxlocker", - "sync15-adapter" + "sync15-adapter", + "sync15-adapter/ffi" ] diff --git a/sync15-adapter/Cargo.toml b/sync15-adapter/Cargo.toml index 97f7ca652e..fe59b6d711 100644 --- a/sync15-adapter/Cargo.toml +++ b/sync15-adapter/Cargo.toml @@ -11,8 +11,8 @@ serde_json = "1.0" url = "1.6.0" reqwest = "0.8.2" error-chain = "0.11" -openssl = "0.10" -hawk = "1.0" +openssl = "0.10.7" +hawk = { git = "https://github.com/eoger/rust-hawk", branch = "use-openssl" } hyper = "0.11" log = "0.4" lazy_static = "1.0" diff --git a/sync15-adapter/examples/boxlocker-parity.rs b/sync15-adapter/examples/boxlocker-parity.rs index f7688f1c85..0c98e5fb29 100644 --- a/sync15-adapter/examples/boxlocker-parity.rs +++ b/sync15-adapter/examples/boxlocker-parity.rs @@ -131,7 +131,10 @@ fn start() -> Result<(), Box> { )?; svc.remote_setup()?; - let passwords = svc.all_records::("passwords")?; + let passwords = svc.all_records::("passwords")? + .into_iter() + .filter_map(|r| r.record()) + .collect::>(); println!("Found {} passwords", passwords.len()); diff --git a/sync15-adapter/ffi/Cargo.toml b/sync15-adapter/ffi/Cargo.toml new file mode 100644 index 0000000000..76c03d590e --- /dev/null +++ b/sync15-adapter/ffi/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sync-adapter-ffi" +version = "0.1.0" +authors = ["Thom Chiovoloni "] + +[lib] +name = "sync_adapter" +crate-type = ["staticlib"] + +[dependencies] +libc = "0.2" + +[dependencies.sync15-adapter] +path = "../" diff --git a/sync15-adapter/ffi/README.md b/sync15-adapter/ffi/README.md new file mode 100644 index 0000000000..9afa06533a --- /dev/null +++ b/sync15-adapter/ffi/README.md @@ -0,0 +1,9 @@ +# Sync 1.5 Client FFI + +This README is shamelessly stolen from the one in the fxa client directory + +## iOS build + +- Make sure you have the nightly compiler in order to get LLVM Bitcode generation. +- Install [cargo-lipo](https://github.com/TimNN/cargo-lipo/#installation). +- Build with: `OPENSSL_DIR=/usr/local/opt/openssl cargo +nightly lipo --release` diff --git a/sync15-adapter/ffi/src/lib.rs b/sync15-adapter/ffi/src/lib.rs new file mode 100644 index 0000000000..3f7d389014 --- /dev/null +++ b/sync15-adapter/ffi/src/lib.rs @@ -0,0 +1,207 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +extern crate sync15_adapter; +extern crate libc; + +use sync15_adapter as sync; +use sync::record_types::{PasswordRecord}; +use std::ffi::{CStr, CString}; +use libc::c_char; +use std::ptr; + +fn c_char_to_string(cchar: *const c_char) -> String { + let c_str = unsafe { CStr::from_ptr(cchar) }; + let r_str = c_str.to_str().unwrap_or(""); + r_str.to_string() +} + +fn string_to_c_char(s: String) -> *mut c_char { + CString::new(s).unwrap().into_raw() +} + +fn opt_string_to_c_char(os: Option) -> *mut c_char { + match os { + Some(s) => string_to_c_char(s), + _ => ptr::null_mut(), + } +} + +#[repr(C)] +pub struct PasswordRecordC { + pub id: *mut c_char, + + /// Might be null! + pub hostname: *mut c_char, + + /// Might be null! + pub form_submit_url: *mut c_char, + pub http_realm: *mut c_char, + + pub username: *mut c_char, + pub password: *mut c_char, + + pub username_field: *mut c_char, + pub password_field: *mut c_char, + + /// In ms since unix epoch + pub time_created: i64, + + /// In ms since unix epoch + pub time_password_changed: i64, + + /// -1 for missing, otherwise in ms_since_unix_epoch + pub time_last_used: i64, + + /// -1 for missing + pub times_used: i64, +} + +unsafe fn drop_cchar_ptr(s: *mut c_char) { + if !s.is_null() { + let _ = CString::from_raw(s); + } +} + +impl Drop for PasswordRecordC { + fn drop(&mut self) { + unsafe { + drop_cchar_ptr(self.id); + drop_cchar_ptr(self.hostname); + drop_cchar_ptr(self.form_submit_url); + drop_cchar_ptr(self.http_realm); + drop_cchar_ptr(self.username); + drop_cchar_ptr(self.password); + drop_cchar_ptr(self.username_field); + drop_cchar_ptr(self.password_field); + } + } +} + +impl From for PasswordRecordC { + fn from(record: PasswordRecord) -> PasswordRecordC { + PasswordRecordC { + id: string_to_c_char(record.id), + hostname: opt_string_to_c_char(record.hostname), + form_submit_url: opt_string_to_c_char(record.form_submit_url), + http_realm: opt_string_to_c_char(record.http_realm), + username: string_to_c_char(record.username), + password: string_to_c_char(record.password), + username_field: string_to_c_char(record.username_field), + password_field: string_to_c_char(record.password_field), + time_created: record.time_created, + time_password_changed: record.time_password_changed, + time_last_used: record.time_last_used.unwrap_or(-1), + times_used: record.times_used.unwrap_or(-1), + } + } +} + +#[no_mangle] +pub extern "C" fn sync15_service_create( + key_id: *const c_char, + access_token: *const c_char, + sync_key: *const c_char, + tokenserver_base_url: *const c_char +) -> *mut sync::Sync15Service { + let params = sync::Sync15ServiceInit { + key_id: c_char_to_string(key_id), + access_token: c_char_to_string(access_token), + sync_key: c_char_to_string(sync_key), + tokenserver_base_url: c_char_to_string(tokenserver_base_url), + }; + let mut boxed = match sync::Sync15Service::new(params) { + Ok(svc) => Box::new(svc), + Err(e) => { + println!("Unexpected error initializing Sync15Service: {}", e); + // TODO: have thoughts about error handling. + return ptr::null_mut(); + } + }; + if let Err(e) = boxed.remote_setup() { + println!("Unexpected error performing remote sync setup: {}", e); + // TODO: have thoughts about error handling here too. + return ptr::null_mut(); + } + Box::into_raw(boxed) +} + +#[no_mangle] +pub extern "C" fn sync15_service_destroy(svc: *mut sync::Sync15Service) { + let _ = unsafe { Box::from_raw(svc) }; +} + +// This is opaque to C +pub struct PasswordCollection { + pub records: Vec, + pub tombstones: Vec, +} + +#[no_mangle] +pub extern "C" fn sync15_service_request_passwords( + svc: *mut sync::Sync15Service +) -> *mut PasswordCollection { + let service = unsafe { &mut *svc }; + let passwords = match service.all_records::("passwords") { + Ok(pws) => pws, + Err(e) => { + // TODO: error handling... + println!("Unexpected error downloading passwords {}", e); + return ptr::null_mut(); + } + }; + let mut tombstones = vec![]; + let mut records = vec![]; + for obj in passwords { + match obj.payload { + sync::Tombstone { id, .. } => tombstones.push(id), + sync::NonTombstone(record) => records.push(record), + } + } + Box::into_raw(Box::new(PasswordCollection { records, tombstones })) +} + +#[no_mangle] +pub extern "C" fn sync15_passwords_destroy(coll: *mut PasswordCollection) { + let _ = unsafe { Box::from_raw(coll) }; +} + +#[no_mangle] +pub extern "C" fn sync15_passwords_tombstone_count(coll: *const PasswordCollection) -> libc::size_t { + let coll = unsafe { &*coll }; + coll.tombstones.len() as libc::size_t +} + +#[no_mangle] +pub extern "C" fn sync15_passwords_record_count(coll: *const PasswordCollection) -> libc::size_t { + let coll = unsafe { &*coll }; + coll.records.len() as libc::size_t +} + +#[no_mangle] +pub extern "C" fn sync15_passwords_get_tombstone_at( + coll: *const PasswordCollection, + index: libc::size_t +) -> *mut c_char { + let coll = unsafe { &*coll }; + opt_string_to_c_char(coll.tombstones.get(index as usize).cloned()) +} + +#[no_mangle] +pub extern "C" fn sync15_passwords_get_record_at( + coll: *const PasswordCollection, + index: libc::size_t +) -> *mut PasswordRecordC { + let coll = unsafe { &*coll }; + match coll.records.get(index as usize) { + Some(r) => Box::into_raw(Box::new(r.clone().into())), + None => ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "C" fn sync15_password_record_destroy(pw: *mut PasswordRecordC) { + // Our drop impl takes care of our strings. + let _ = unsafe { Box::from_raw(pw) }; +} diff --git a/sync15-adapter/ffi/sync_adapter.h b/sync15-adapter/ffi/sync_adapter.h new file mode 100644 index 0000000000..db3f4a9b8f --- /dev/null +++ b/sync15-adapter/ffi/sync_adapter.h @@ -0,0 +1,63 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef SYNC_ADAPTER_15_H +#define SYNC_ADAPTER_15_H +// size_t +#include +// int64_t +#include + +typedef struct sync15_PasswordRecord sync15_PasswordRecord; +typedef struct sync15_PasswordCollection sync15_PasswordCollection; +typedef struct sync15_Service sync15_Service; + +struct sync15_PasswordRecord { + const char* id; + // Might be null! + const char* hostname; + // Might be null! + const char* form_submit_url; + const char* http_realm; + + const char* username; + const char* password; + + const char* username_field; + const char* password_field; + + // In ms since unix epoch + int64_t time_created; + + // In ms since unix epoch + int64_t time_password_changed; + + // -1 for missing, otherwise in ms_since_unix_epoch + int64_t time_last_used; + + // -1 for missing + int64_t times_used; +}; + +sync15_Service *sync15_service_create(const char* key_id, + const char* access_token, + const char* sync_key, + const char* tokenserver_base_url); + +void sync15_service_destroy(sync15_Service* svc); + +sync15_PasswordCollection* sync15_service_request_passwords(sync15_Service* svc); +void sync15_passwords_destroy(sync15_PasswordCollection *passwords); + +size_t sync15_passwords_record_count(const sync15_PasswordCollection* passwords); +size_t sync15_passwords_tombstone_count(const sync15_PasswordCollection* passwords); + +// Caller frees! Returns null if index > sync15_passwords_tombstone_count(passwords) +char *sync15_passwords_get_tombstone_at(const sync15_PasswordCollection* pws, size_t i); + +// Caller frees (via sync15_password_record_free) Returns null if index > sync15_passwords_record_count(pws) +sync15_PasswordRecord* sync15_passwords_get_record_at(const sync15_PasswordCollection* pws, size_t i); + +void sync15_password_record_destroy(sync15_PasswordRecord *record); + +#endif diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 874531360d..5ae7670df8 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -206,18 +206,14 @@ impl Sync15Service { .key_for_collection(collection)) } - pub fn all_records(&mut self, collection: &str) -> error::Result>> where T: Sync15Record { + pub fn all_records(&mut self, collection: &str) -> + error::Result>>> where T: Sync15Record { let key = self.key_for_collection(collection)?; let mut resp = self.collection_request(Method::Get, CollectionRequest::new(collection).full())?; let records: Vec> = resp.json()?; - let mut result = Vec::with_capacity(records.len()); - for record in records { - let decrypted: BsoRecord> = record.decrypt(key)?; - if let Some(record) = decrypted.record() { - result.push(record); - } - } - Ok(result) + records.into_iter() + .map(|record| record.decrypt::>(key)) + .collect() } fn update_timestamp(&self, hs: &header::Headers) { diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs index 88aa436e0d..d5c7ef4bf6 100644 --- a/sync15-adapter/src/token.rs +++ b/sync15-adapter/src/token.rs @@ -8,6 +8,7 @@ use reqwest::{Client, Request, Url}; use hyper::header::{Authorization, Bearer}; use error::{self, Result}; use std::fmt; +use openssl::hash::MessageDigest; use std::borrow::{Borrow, Cow}; use util::ServerTimestamp; @@ -96,7 +97,7 @@ impl TokenserverClient { "Missing or corrupted X-Timestamp header from token server"))?; let credentials = hawk::Credentials { id: token.id.clone(), - key: hawk::Key::new(token.key.as_bytes(), &hawk::SHA256), + key: hawk::Key::new(token.key.as_bytes(), MessageDigest::sha256())?, }; Ok(TokenserverClient { token, From 88da47546bc2b280d5497cd4059f77ea2d2f6f83 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 2 May 2018 12:35:12 -0700 Subject: [PATCH 22/23] Fix warnings and stale comments --- sync15-adapter/src/lib.rs | 3 ++- sync15-adapter/src/request.rs | 1 - sync15-adapter/src/token.rs | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index 5ae7670df8..be3c9d608b 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -22,7 +22,8 @@ extern crate serde_derive; #[macro_use] extern crate log; -#[macro_use] +// Right now we only need the `json!` macro in tests, and a raw `#[macro_use]` gives us a warning +#[cfg_attr(test, macro_use)] extern crate serde_json; #[macro_use] diff --git a/sync15-adapter/src/request.rs b/sync15-adapter/src/request.rs index da569af328..19b94398d1 100644 --- a/sync15-adapter/src/request.rs +++ b/sync15-adapter/src/request.rs @@ -287,7 +287,6 @@ pub enum BatchState { #[derive(Debug)] pub struct PostQueue { poster: Post, - // XXX: It's cludgey that this needs to be pub just so we can get at the data it saves... on_response: OnResponse, post_limits: LimitTracker, batch_limits: LimitTracker, diff --git a/sync15-adapter/src/token.rs b/sync15-adapter/src/token.rs index d5c7ef4bf6..d368884d62 100644 --- a/sync15-adapter/src/token.rs +++ b/sync15-adapter/src/token.rs @@ -8,7 +8,6 @@ use reqwest::{Client, Request, Url}; use hyper::header::{Authorization, Bearer}; use error::{self, Result}; use std::fmt; -use openssl::hash::MessageDigest; use std::borrow::{Borrow, Cow}; use util::ServerTimestamp; @@ -97,7 +96,7 @@ impl TokenserverClient { "Missing or corrupted X-Timestamp header from token server"))?; let credentials = hawk::Credentials { id: token.id.clone(), - key: hawk::Key::new(token.key.as_bytes(), MessageDigest::sha256())?, + key: hawk::Key::new(token.key.as_bytes(), hawk::Digest::sha256())?, }; Ok(TokenserverClient { token, From bcbb71c2652f12d3e274691303688adf78f3f201 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Wed, 2 May 2018 13:03:34 -0700 Subject: [PATCH 23/23] Clean up file structure a bit based on review - Tombstone stuff is now in tombstone.rs - record_types.rs just includes type definitions for known record formats. - service code is now in service.rs - lib.rs just has crate decls, module decls, and exports. --- sync15-adapter/examples/boxlocker-parity.rs | 3 +- sync15-adapter/ffi/src/lib.rs | 3 +- sync15-adapter/src/lib.rs | 306 +------------------- sync15-adapter/src/record_types.rs | 176 +---------- sync15-adapter/src/service.rs | 306 ++++++++++++++++++++ sync15-adapter/src/tombstone.rs | 179 ++++++++++++ 6 files changed, 497 insertions(+), 476 deletions(-) create mode 100644 sync15-adapter/src/service.rs create mode 100644 sync15-adapter/src/tombstone.rs diff --git a/sync15-adapter/examples/boxlocker-parity.rs b/sync15-adapter/examples/boxlocker-parity.rs index 0c98e5fb29..73f3ac2dc8 100644 --- a/sync15-adapter/examples/boxlocker-parity.rs +++ b/sync15-adapter/examples/boxlocker-parity.rs @@ -1,5 +1,5 @@ -extern crate sync15_adapter; +extern crate sync15_adapter as sync; extern crate error_chain; extern crate url; extern crate base64; @@ -19,7 +19,6 @@ use std::process; use std::time; use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; -use sync15_adapter as sync; #[derive(Debug, Deserialize)] struct OAuthCredentials { diff --git a/sync15-adapter/ffi/src/lib.rs b/sync15-adapter/ffi/src/lib.rs index 3f7d389014..812d7b0b14 100644 --- a/sync15-adapter/ffi/src/lib.rs +++ b/sync15-adapter/ffi/src/lib.rs @@ -2,10 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -extern crate sync15_adapter; +extern crate sync15_adapter as sync; extern crate libc; -use sync15_adapter as sync; use sync::record_types::{PasswordRecord}; use std::ffi::{CStr, CString}; use libc::c_char; diff --git a/sync15-adapter/src/lib.rs b/sync15-adapter/src/lib.rs index be3c9d608b..47ef63a838 100644 --- a/sync15-adapter/src/lib.rs +++ b/sync15-adapter/src/lib.rs @@ -40,303 +40,13 @@ pub mod token; pub mod collection_keys; pub mod util; pub mod request; +pub mod service; +pub mod tombstone; -pub use MaybeTombstone::*; - -use util::ServerTimestamp; - -use std::cell::Cell; -use std::time::{Duration}; -use std::collections::{HashMap, HashSet}; - -use key_bundle::KeyBundle; -use reqwest::{ - Client, - Request, - Response, - Url, - header::{self, Accept} -}; -use hyper::Method; -use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; -use record_types::{MaybeTombstone, MetaGlobalRecord}; -use collection_keys::CollectionKeys; -use request::{ - CollectionRequest, - InfoConfiguration, - XWeaveTimestamp, - XIfUnmodifiedSince, - PostResponse, - BatchPoster, - PostQueue, - PostResponseHandler, - NormalResponseHandler, -}; - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Sync15ServiceInit { - pub key_id: String, - pub access_token: String, - pub sync_key: String, - pub tokenserver_base_url: String, -} - -#[derive(Debug)] -pub struct Sync15Service { - init_params: Sync15ServiceInit, - root_key: KeyBundle, - client: Client, - // We update this when we make requests - last_server_time: Cell, - tsc: token::TokenserverClient, - - keys: Option, - server_config: Option, - last_sync_remote: HashMap, -} - -impl Sync15Service { - pub fn new(init_params: Sync15ServiceInit) -> error::Result { - let root_key = KeyBundle::from_ksync_base64(&init_params.sync_key)?; - let client = Client::builder().timeout(Duration::from_secs(30)).build()?; - // TODO Should we be doing this here? What if we get backoff? Who handles that? - let tsc = token::TokenserverClient::new(&client, - &init_params.tokenserver_base_url, - init_params.access_token.clone(), - init_params.key_id.clone())?; - let timestamp = tsc.server_timestamp(); - Ok(Sync15Service { - client, - init_params, - root_key, - tsc, - last_server_time: Cell::new(timestamp), - keys: None, - server_config: None, - last_sync_remote: HashMap::new(), - }) - } - - #[inline] - fn authorized(&self, mut req: Request) -> error::Result { - let header = self.tsc.authorization(&req)?; - req.headers_mut().set(header); - Ok(req) - } - - // TODO: probably want a builder-like API to do collection requests (e.g. something - // that occupies roughly the same conceptual role as the Collection class in desktop) - fn build_request(&self, method: Method, url: Url) -> error::Result { - self.authorized(self.client.request(method, url).header(Accept::json()).build()?) - } - - fn relative_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { - let s = self.tsc.token().api_endpoint.clone() + "/"; - let url = Url::parse(&s)?.join(relative_path.as_ref())?; - Ok(self.make_storage_request(method, url)?) - } - - fn make_storage_request(&self, method: Method, url: Url) -> error::Result { - // I'm shocked that method isn't Copy... - Ok(self.exec_request(self.build_request(method.clone(), url)?, true)?) - } - - fn exec_request(&self, req: Request, require_success: bool) -> error::Result { - let resp = self.client.execute(req)?; - - self.update_timestamp(resp.headers()); - - if require_success && !resp.status().is_success() { - error!("HTTP error {} ({}) during storage request to {}", - resp.status().as_u16(), resp.status(), resp.url().path()); - bail!(error::ErrorKind::StorageHttpError( - resp.status(), resp.url().path().into())); - } - - // TODO: - // - handle backoff - // - x-weave-quota? - // - ... almost certainly other things too... - - Ok(resp) - } - - fn collection_request(&self, method: Method, r: &CollectionRequest) -> error::Result { - self.make_storage_request(method.clone(), - r.build_url(Url::parse(&self.tsc.token().api_endpoint)?)?) - } - - fn fetch_info(&self, path: &str) -> error::Result where for <'a> T: serde::de::Deserialize<'a> { - let mut resp = self.relative_storage_request(Method::Get, path)?; - let result: T = resp.json()?; - Ok(result) - } +// Re-export some of the types callers are likely to want for convenience. +pub use bso_record::{BsoRecord, Sync15Record}; +pub use tombstone::{MaybeTombstone, Tombstone, NonTombstone}; +pub use service::{Sync15ServiceInit, Sync15Service, CollectionUpdate}; +pub use error::{Result, Error, ErrorKind}; - pub fn remote_setup(&mut self) -> error::Result<()> { - let server_config = self.fetch_info::("info/configuration")?; - self.server_config = Some(server_config); - let mut resp = match self.relative_storage_request(Method::Get, "storage/meta/global") { - Ok(r) => r, - // This is gross, but at least it works. Replace 404s on meta/global with NoMetaGlobal. - Err(error::Error(error::ErrorKind::StorageHttpError(hyper::StatusCode::NotFound, ..), _)) => - bail!(error::ErrorKind::NoMetaGlobal), - Err(e) => return Err(e), - }; - // Note: meta/global is not encrypted! - let meta_global: BsoRecord = resp.json()?; - info!("Meta global: {:?}", meta_global.payload); - let collections = self.fetch_info::>("info/collections")?; - self.update_keys(&collections)?; - self.last_sync_remote = collections; - Ok(()) - } - - fn update_keys(&mut self, _info_collections: &HashMap) -> error::Result<()> { - // TODO: if info/collections says we should, upload keys. - // TODO: This should be handled in collection_keys.rs, which should track modified time, etc. - let mut keys_resp = self.relative_storage_request(Method::Get, "storage/crypto/keys")?; - let keys: BsoRecord = keys_resp.json()?; - self.keys = Some(CollectionKeys::from_encrypted_bso(keys, &self.root_key)?); - // TODO: error handling... key upload? - Ok(()) - } - - pub fn key_for_collection(&self, collection: &str) -> error::Result<&KeyBundle> { - Ok(self.keys.as_ref() - .ok_or_else(|| error::unexpected("Don't have keys (yet?)"))? - .key_for_collection(collection)) - } - - pub fn all_records(&mut self, collection: &str) -> - error::Result>>> where T: Sync15Record { - let key = self.key_for_collection(collection)?; - let mut resp = self.collection_request(Method::Get, CollectionRequest::new(collection).full())?; - let records: Vec> = resp.json()?; - records.into_iter() - .map(|record| record.decrypt::>(key)) - .collect() - } - - fn update_timestamp(&self, hs: &header::Headers) { - if let Some(ts) = hs.get::().map(|h| **h) { - self.last_server_time.set(ts); - } else { - // Should we complain more here? - warn!("No X-Weave-Timestamp from storage server!"); - } - } - - pub fn last_modified(&self, coll: &str) -> Option { - self.last_sync_remote.get(coll).cloned() - } - - pub fn last_modified_or_zero(&self, coll: &str) -> ServerTimestamp { - self.last_modified(coll).unwrap_or(util::SERVER_EPOCH) - } - - fn new_post_queue<'a, F: PostResponseHandler>(&'a self, coll: &str, lm: Option, on_response: F) - -> error::Result, F>> { - let ts = lm.unwrap_or_else(|| self.last_modified_or_zero(&coll)); - let pw = PostWrapper { svc: self, coll: coll.into() }; - Ok(PostQueue::new(self.server_config.as_ref().unwrap(), ts, pw, on_response)) - } -} - -struct PostWrapper<'a> { - svc: &'a Sync15Service, - coll: String, -} - -impl<'a> BatchPoster for PostWrapper<'a> { - fn post(&self, - bytes: &[u8], - xius: ServerTimestamp, - batch: Option, - commit: bool, - _: &PostQueue) -> error::Result - { - let url = CollectionRequest::new(self.coll.clone()) - .batch(batch) - .commit(commit) - .build_url(Url::parse(&self.svc.tsc.token().api_endpoint)?)?; - - let mut req = self.svc.build_request(Method::Post, url)?; - req.headers_mut().set(hyper::header::ContentType::json()); - req.headers_mut().set(XIfUnmodifiedSince(xius)); - // It's very annoying that we need to copy the body here, the request - // shouldn't need to take ownership of it... - *req.body_mut() = Some(Vec::from(bytes).into()); - let mut resp = self.svc.exec_request(req, false)?; - Ok(PostResponse::from_response(&mut resp)?) - } -} - -#[derive(Debug, Clone)] -pub struct CollectionUpdate<'a, T> { - svc: &'a Sync15Service, - last_sync: ServerTimestamp, - to_update: Vec>, - allow_dropped_records: bool, - queued_ids: HashSet -} - -impl<'a, T> CollectionUpdate<'a, T> where T: Sync15Record { - pub fn new(svc: &'a Sync15Service, allow_dropped_records: bool) -> CollectionUpdate<'a, T> { - let coll = T::collection_tag(); - let ts = svc.last_modified_or_zero(coll); - CollectionUpdate { - svc, - last_sync: ts, - to_update: vec![], - allow_dropped_records, - queued_ids: HashSet::new(), - } - } - - pub fn add(&mut self, rec_or_tombstone: MaybeTombstone) { - // Block to limit scope of the `id` borrow. - { - let id = rec_or_tombstone.record_id(); - // Should this be an Err and not an assertion? - assert!(!self.queued_ids.contains(id), - "Attempt to update ID multiple times in the same batch {}", id); - self.queued_ids.insert(id.into()); - } - self.to_update.push(rec_or_tombstone); - } - - pub fn add_record(&mut self, record: T) { - self.add(NonTombstone(record)); - } - - pub fn add_tombstone(&mut self, id: String) { - self.add(MaybeTombstone::tombstone(id)); - } - - /// Returns a list of the IDs that failed if allowed_dropped_records is true, otherwise - /// returns an empty vec. - pub fn upload(self) -> error::Result<(Vec, Vec)> { - let mut failed = vec![]; - let key = self.svc.key_for_collection(T::collection_tag())?; - let mut q = self.svc.new_post_queue(T::collection_tag(), Some(self.last_sync), - NormalResponseHandler::new(self.allow_dropped_records))?; - - for record in self.to_update.into_iter() { - let record_cleartext: BsoRecord> = record.into(); - let encrypted = record_cleartext.encrypt(key)?; - let enqueued = q.enqueue(&encrypted)?; - if !enqueued && !self.allow_dropped_records { - bail!(error::ErrorKind::RecordTooLargeError); - } - } - - q.flush(true)?; - let (successful_ids, mut failed_ids) = q.successful_and_failed_ids(); - failed_ids.append(&mut failed); - if !self.allow_dropped_records { - assert_eq!(failed_ids.len(), 0, - "Bug: Should have failed by now if we aren't allowing dropped records"); - } - Ok((successful_ids, failed_ids)) - } -} +pub use MaybeTombstone::*; diff --git a/sync15-adapter/src/record_types.rs b/sync15-adapter/src/record_types.rs index 872c301411..a0872c5ea3 100644 --- a/sync15-adapter/src/record_types.rs +++ b/sync15-adapter/src/record_types.rs @@ -2,95 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use bso_record::{BsoRecord, Sync15Record}; +use bso_record::Sync15Record; use std::collections::HashMap; -pub use MaybeTombstone::*; - -#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] -#[serde(untagged)] -pub enum MaybeTombstone { - Tombstone { id: String, deleted: bool }, - NonTombstone(T) -} - -impl MaybeTombstone { - #[inline] - pub fn tombstone>(id: R) -> MaybeTombstone { - Tombstone { id: id.into(), deleted: true } - } - - #[inline] - pub fn is_tombstone(&self) -> bool { - match self { - &NonTombstone(_) => false, - _ => true - } - } - - #[inline] - pub fn unwrap(self) -> T { - match self { - NonTombstone(record) => record, - _ => panic!("called `MaybeTombstone::unwrap()` on a Tombstone!"), - } - } - - #[inline] - pub fn expect(self, msg: &str) -> T { - match self { - NonTombstone(record) => record, - _ => panic!("{}", msg), - } - } - - #[inline] - pub fn ok_or(self, err: E) -> ::std::result::Result { - match self { - NonTombstone(record) => Ok(record), - _ => Err(err) - } - } - - #[inline] - pub fn record(self) -> Option { - match self { - NonTombstone(record) => Some(record), - _ => None - } - } -} - -impl Sync15Record for MaybeTombstone where T: Sync15Record { - fn collection_tag() -> &'static str { T::collection_tag() } - fn ttl() -> Option { T::ttl() } - fn record_id(&self) -> &str { - match self { - &Tombstone { ref id, .. } => id, - &NonTombstone(ref record) => record.record_id() - } - } - fn sortindex(&self) -> Option { - match self { - &Tombstone { .. } => None, - &NonTombstone(ref record) => record.sortindex() - } - } -} - -impl BsoRecord> { - #[inline] - pub fn is_tombstone(&self) -> bool { - self.payload.is_tombstone() - } - - #[inline] - pub fn record(self) -> Option> where T: Clone { - self.map_payload(|payload| payload.record()).transpose() - } -} - -pub type MaybeTombstoneRecord = BsoRecord>; +// Known record formats. #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -152,90 +67,3 @@ impl Sync15Record for MetaGlobalRecord { fn collection_tag() -> &'static str { "meta" } fn record_id(&self) -> &str { "global" } } - -#[cfg(test)] -mod tests { - - use super::*; - use key_bundle::KeyBundle; - use util::ServerTimestamp; - use serde_json; - - #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] - struct DummyRecord { - id: String, - age: i64, - meta: String, - } - - impl Sync15Record for DummyRecord { - fn collection_tag() -> &'static str { "dummy" } - fn record_id(&self) -> &str { &self.id } - } - - #[test] - fn test_roundtrip_crypt_tombstone() { - let orig_record: MaybeTombstoneRecord = BsoRecord { - id: "aaaaaaaaaaaa".into(), - collection: "dummy".into(), - modified: ServerTimestamp(1234.0), - sortindex: None, - ttl: None, - payload: MaybeTombstone::tombstone("aaaaaaaaaaaa") - }; - - assert!(orig_record.is_tombstone()); - - let keybundle = KeyBundle::new_random().unwrap(); - - let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); - - assert!(keybundle.verify_hmac_string( - &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); - - // While we're here, check on EncryptedPayload::serialized_len - let val_rec = serde_json::from_str::( - &serde_json::to_string(&encrypted).unwrap()).unwrap(); - assert_eq!(encrypted.payload.serialized_len(), - val_rec["payload"].as_str().unwrap().len()); - - let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); - assert!(decrypted.is_tombstone()); - assert_eq!(decrypted, orig_record); - } - - #[test] - fn test_roundtrip_crypt_record() { - let orig_record: MaybeTombstoneRecord = BsoRecord { - id: "aaaaaaaaaaaa".into(), - collection: "dummy".into(), - modified: ServerTimestamp(1234.0), - sortindex: None, - ttl: None, - payload: NonTombstone(DummyRecord { - id: "aaaaaaaaaaaa".into(), - age: 105, - meta: "data".into() - }) - }; - - assert!(!orig_record.is_tombstone()); - - let keybundle = KeyBundle::new_random().unwrap(); - - let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); - - assert!(keybundle.verify_hmac_string( - &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); - - // While we're here, check on EncryptedPayload::serialized_len - let val_rec = serde_json::from_str::( - &serde_json::to_string(&encrypted).unwrap()).unwrap(); - assert_eq!(encrypted.payload.serialized_len(), - val_rec["payload"].as_str().unwrap().len()); - - let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); - assert!(!decrypted.is_tombstone()); - assert_eq!(decrypted, orig_record); - } -} diff --git a/sync15-adapter/src/service.rs b/sync15-adapter/src/service.rs new file mode 100644 index 0000000000..02ac7cd04b --- /dev/null +++ b/sync15-adapter/src/service.rs @@ -0,0 +1,306 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +use std::cell::Cell; +use std::time::{Duration}; +use std::collections::{HashMap, HashSet}; + +use reqwest::{ + Client, + Request, + Response, + Url, + header::{self, Accept} +}; +use hyper::{Method, StatusCode}; +use serde; + +use util::{ServerTimestamp, SERVER_EPOCH}; +use token; +use error; +use key_bundle::KeyBundle; +use bso_record::{BsoRecord, Sync15Record, EncryptedPayload}; +use tombstone::{MaybeTombstone, NonTombstone}; +use record_types::MetaGlobalRecord; +use collection_keys::CollectionKeys; +use request::{ + CollectionRequest, + InfoConfiguration, + XWeaveTimestamp, + XIfUnmodifiedSince, + PostResponse, + BatchPoster, + PostQueue, + PostResponseHandler, + NormalResponseHandler, +}; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Sync15ServiceInit { + pub key_id: String, + pub access_token: String, + pub sync_key: String, + pub tokenserver_base_url: String, +} + +#[derive(Debug)] +pub struct Sync15Service { + init_params: Sync15ServiceInit, + root_key: KeyBundle, + client: Client, + // We update this when we make requests + last_server_time: Cell, + tsc: token::TokenserverClient, + + keys: Option, + server_config: Option, + last_sync_remote: HashMap, +} + +impl Sync15Service { + pub fn new(init_params: Sync15ServiceInit) -> error::Result { + let root_key = KeyBundle::from_ksync_base64(&init_params.sync_key)?; + let client = Client::builder().timeout(Duration::from_secs(30)).build()?; + // TODO Should we be doing this here? What if we get backoff? Who handles that? + let tsc = token::TokenserverClient::new(&client, + &init_params.tokenserver_base_url, + init_params.access_token.clone(), + init_params.key_id.clone())?; + let timestamp = tsc.server_timestamp(); + Ok(Sync15Service { + client, + init_params, + root_key, + tsc, + last_server_time: Cell::new(timestamp), + keys: None, + server_config: None, + last_sync_remote: HashMap::new(), + }) + } + + #[inline] + fn authorized(&self, mut req: Request) -> error::Result { + let header = self.tsc.authorization(&req)?; + req.headers_mut().set(header); + Ok(req) + } + + // TODO: probably want a builder-like API to do collection requests (e.g. something + // that occupies roughly the same conceptual role as the Collection class in desktop) + fn build_request(&self, method: Method, url: Url) -> error::Result { + self.authorized(self.client.request(method, url).header(Accept::json()).build()?) + } + + fn relative_storage_request(&self, method: Method, relative_path: T) -> error::Result where T: AsRef { + let s = self.tsc.token().api_endpoint.clone() + "/"; + let url = Url::parse(&s)?.join(relative_path.as_ref())?; + Ok(self.make_storage_request(method, url)?) + } + + fn make_storage_request(&self, method: Method, url: Url) -> error::Result { + // I'm shocked that method isn't Copy... + Ok(self.exec_request(self.build_request(method.clone(), url)?, true)?) + } + + fn exec_request(&self, req: Request, require_success: bool) -> error::Result { + let resp = self.client.execute(req)?; + + self.update_timestamp(resp.headers()); + + if require_success && !resp.status().is_success() { + error!("HTTP error {} ({}) during storage request to {}", + resp.status().as_u16(), resp.status(), resp.url().path()); + bail!(error::ErrorKind::StorageHttpError( + resp.status(), resp.url().path().into())); + } + + // TODO: + // - handle backoff + // - x-weave-quota? + // - ... almost certainly other things too... + + Ok(resp) + } + + fn collection_request(&self, method: Method, r: &CollectionRequest) -> error::Result { + self.make_storage_request(method.clone(), + r.build_url(Url::parse(&self.tsc.token().api_endpoint)?)?) + } + + fn fetch_info(&self, path: &str) -> error::Result where for <'a> T: serde::de::Deserialize<'a> { + let mut resp = self.relative_storage_request(Method::Get, path)?; + let result: T = resp.json()?; + Ok(result) + } + + pub fn remote_setup(&mut self) -> error::Result<()> { + let server_config = self.fetch_info::("info/configuration")?; + self.server_config = Some(server_config); + let mut resp = match self.relative_storage_request(Method::Get, "storage/meta/global") { + Ok(r) => r, + // This is gross, but at least it works. Replace 404s on meta/global with NoMetaGlobal. + Err(error::Error(error::ErrorKind::StorageHttpError(StatusCode::NotFound, ..), _)) => + bail!(error::ErrorKind::NoMetaGlobal), + Err(e) => return Err(e), + }; + // Note: meta/global is not encrypted! + let meta_global: BsoRecord = resp.json()?; + info!("Meta global: {:?}", meta_global.payload); + let collections = self.fetch_info::>("info/collections")?; + self.update_keys(&collections)?; + self.last_sync_remote = collections; + Ok(()) + } + + fn update_keys(&mut self, _info_collections: &HashMap) -> error::Result<()> { + // TODO: if info/collections says we should, upload keys. + // TODO: This should be handled in collection_keys.rs, which should track modified time, etc. + let mut keys_resp = self.relative_storage_request(Method::Get, "storage/crypto/keys")?; + let keys: BsoRecord = keys_resp.json()?; + self.keys = Some(CollectionKeys::from_encrypted_bso(keys, &self.root_key)?); + // TODO: error handling... key upload? + Ok(()) + } + + pub fn key_for_collection(&self, collection: &str) -> error::Result<&KeyBundle> { + Ok(self.keys.as_ref() + .ok_or_else(|| error::unexpected("Don't have keys (yet?)"))? + .key_for_collection(collection)) + } + + pub fn all_records(&mut self, collection: &str) -> + error::Result>>> where T: Sync15Record { + let key = self.key_for_collection(collection)?; + let mut resp = self.collection_request(Method::Get, CollectionRequest::new(collection).full())?; + let records: Vec> = resp.json()?; + records.into_iter() + .map(|record| record.decrypt::>(key)) + .collect() + } + + fn update_timestamp(&self, hs: &header::Headers) { + if let Some(ts) = hs.get::().map(|h| **h) { + self.last_server_time.set(ts); + } else { + // Should we complain more here? + warn!("No X-Weave-Timestamp from storage server!"); + } + } + + pub fn last_modified(&self, coll: &str) -> Option { + self.last_sync_remote.get(coll).cloned() + } + + pub fn last_modified_or_zero(&self, coll: &str) -> ServerTimestamp { + self.last_modified(coll).unwrap_or(SERVER_EPOCH) + } + + fn new_post_queue<'a, F: PostResponseHandler>(&'a self, coll: &str, lm: Option, on_response: F) + -> error::Result, F>> { + let ts = lm.unwrap_or_else(|| self.last_modified_or_zero(&coll)); + let pw = PostWrapper { svc: self, coll: coll.into() }; + Ok(PostQueue::new(self.server_config.as_ref().unwrap(), ts, pw, on_response)) + } +} + +struct PostWrapper<'a> { + svc: &'a Sync15Service, + coll: String, +} + +impl<'a> BatchPoster for PostWrapper<'a> { + fn post(&self, + bytes: &[u8], + xius: ServerTimestamp, + batch: Option, + commit: bool, + _: &PostQueue) -> error::Result + { + let url = CollectionRequest::new(self.coll.clone()) + .batch(batch) + .commit(commit) + .build_url(Url::parse(&self.svc.tsc.token().api_endpoint)?)?; + + let mut req = self.svc.build_request(Method::Post, url)?; + req.headers_mut().set(header::ContentType::json()); + req.headers_mut().set(XIfUnmodifiedSince(xius)); + // It's very annoying that we need to copy the body here, the request + // shouldn't need to take ownership of it... + *req.body_mut() = Some(Vec::from(bytes).into()); + let mut resp = self.svc.exec_request(req, false)?; + Ok(PostResponse::from_response(&mut resp)?) + } +} + +#[derive(Debug, Clone)] +pub struct CollectionUpdate<'a, T> { + svc: &'a Sync15Service, + last_sync: ServerTimestamp, + to_update: Vec>, + allow_dropped_records: bool, + queued_ids: HashSet +} + +impl<'a, T> CollectionUpdate<'a, T> where T: Sync15Record { + pub fn new(svc: &'a Sync15Service, allow_dropped_records: bool) -> CollectionUpdate<'a, T> { + let coll = T::collection_tag(); + let ts = svc.last_modified_or_zero(coll); + CollectionUpdate { + svc, + last_sync: ts, + to_update: vec![], + allow_dropped_records, + queued_ids: HashSet::new(), + } + } + + pub fn add(&mut self, rec_or_tombstone: MaybeTombstone) { + // Block to limit scope of the `id` borrow. + { + let id = rec_or_tombstone.record_id(); + // Should this be an Err and not an assertion? + assert!(!self.queued_ids.contains(id), + "Attempt to update ID multiple times in the same batch {}", id); + self.queued_ids.insert(id.into()); + } + self.to_update.push(rec_or_tombstone); + } + + pub fn add_record(&mut self, record: T) { + self.add(NonTombstone(record)); + } + + pub fn add_tombstone(&mut self, id: String) { + self.add(MaybeTombstone::tombstone(id)); + } + + /// Returns a list of the IDs that failed if allowed_dropped_records is true, otherwise + /// returns an empty vec. + pub fn upload(self) -> error::Result<(Vec, Vec)> { + let mut failed = vec![]; + let key = self.svc.key_for_collection(T::collection_tag())?; + let mut q = self.svc.new_post_queue(T::collection_tag(), Some(self.last_sync), + NormalResponseHandler::new(self.allow_dropped_records))?; + + for record in self.to_update.into_iter() { + let record_cleartext: BsoRecord> = record.into(); + let encrypted = record_cleartext.encrypt(key)?; + let enqueued = q.enqueue(&encrypted)?; + if !enqueued && !self.allow_dropped_records { + bail!(error::ErrorKind::RecordTooLargeError); + } + } + + q.flush(true)?; + let (successful_ids, mut failed_ids) = q.successful_and_failed_ids(); + failed_ids.append(&mut failed); + if !self.allow_dropped_records { + assert_eq!(failed_ids.len(), 0, + "Bug: Should have failed by now if we aren't allowing dropped records"); + } + Ok((successful_ids, failed_ids)) + } +} diff --git a/sync15-adapter/src/tombstone.rs b/sync15-adapter/src/tombstone.rs new file mode 100644 index 0000000000..d15a027da5 --- /dev/null +++ b/sync15-adapter/src/tombstone.rs @@ -0,0 +1,179 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use bso_record::{BsoRecord, Sync15Record}; + +pub use MaybeTombstone::*; + +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +#[serde(untagged)] +pub enum MaybeTombstone { + Tombstone { id: String, deleted: bool }, + NonTombstone(T) +} + +impl MaybeTombstone { + #[inline] + pub fn tombstone>(id: R) -> MaybeTombstone { + Tombstone { id: id.into(), deleted: true } + } + + #[inline] + pub fn is_tombstone(&self) -> bool { + match self { + &NonTombstone(_) => false, + _ => true + } + } + + #[inline] + pub fn unwrap(self) -> T { + match self { + NonTombstone(record) => record, + _ => panic!("called `MaybeTombstone::unwrap()` on a Tombstone!"), + } + } + + #[inline] + pub fn expect(self, msg: &str) -> T { + match self { + NonTombstone(record) => record, + _ => panic!("{}", msg), + } + } + + #[inline] + pub fn ok_or(self, err: E) -> ::std::result::Result { + match self { + NonTombstone(record) => Ok(record), + _ => Err(err) + } + } + + #[inline] + pub fn record(self) -> Option { + match self { + NonTombstone(record) => Some(record), + _ => None + } + } +} + +impl Sync15Record for MaybeTombstone where T: Sync15Record { + fn collection_tag() -> &'static str { T::collection_tag() } + fn ttl() -> Option { T::ttl() } + fn record_id(&self) -> &str { + match self { + &Tombstone { ref id, .. } => id, + &NonTombstone(ref record) => record.record_id() + } + } + fn sortindex(&self) -> Option { + match self { + &Tombstone { .. } => None, + &NonTombstone(ref record) => record.sortindex() + } + } +} + +impl BsoRecord> { + #[inline] + pub fn is_tombstone(&self) -> bool { + self.payload.is_tombstone() + } + + #[inline] + pub fn record(self) -> Option> where T: Clone { + self.map_payload(|payload| payload.record()).transpose() + } +} + +pub type MaybeTombstoneRecord = BsoRecord>; + +#[cfg(test)] +mod tests { + + use super::*; + use key_bundle::KeyBundle; + use util::ServerTimestamp; + use serde_json; + + #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] + struct DummyRecord { + id: String, + age: i64, + meta: String, + } + + impl Sync15Record for DummyRecord { + fn collection_tag() -> &'static str { "dummy" } + fn record_id(&self) -> &str { &self.id } + } + + #[test] + fn test_roundtrip_crypt_tombstone() { + let orig_record: MaybeTombstoneRecord = BsoRecord { + id: "aaaaaaaaaaaa".into(), + collection: "dummy".into(), + modified: ServerTimestamp(1234.0), + sortindex: None, + ttl: None, + payload: MaybeTombstone::tombstone("aaaaaaaaaaaa") + }; + + assert!(orig_record.is_tombstone()); + + let keybundle = KeyBundle::new_random().unwrap(); + + let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); + + assert!(keybundle.verify_hmac_string( + &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + + // While we're here, check on EncryptedPayload::serialized_len + let val_rec = serde_json::from_str::( + &serde_json::to_string(&encrypted).unwrap()).unwrap(); + assert_eq!(encrypted.payload.serialized_len(), + val_rec["payload"].as_str().unwrap().len()); + + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); + assert!(decrypted.is_tombstone()); + assert_eq!(decrypted, orig_record); + } + + #[test] + fn test_roundtrip_crypt_record() { + let orig_record: MaybeTombstoneRecord = BsoRecord { + id: "aaaaaaaaaaaa".into(), + collection: "dummy".into(), + modified: ServerTimestamp(1234.0), + sortindex: None, + ttl: None, + payload: NonTombstone(DummyRecord { + id: "aaaaaaaaaaaa".into(), + age: 105, + meta: "data".into() + }) + }; + + assert!(!orig_record.is_tombstone()); + + let keybundle = KeyBundle::new_random().unwrap(); + + let encrypted = orig_record.clone().encrypt(&keybundle).unwrap(); + + assert!(keybundle.verify_hmac_string( + &encrypted.payload.hmac, &encrypted.payload.ciphertext).unwrap()); + + // While we're here, check on EncryptedPayload::serialized_len + let val_rec = serde_json::from_str::( + &serde_json::to_string(&encrypted).unwrap()).unwrap(); + assert_eq!(encrypted.payload.serialized_len(), + val_rec["payload"].as_str().unwrap().len()); + + let decrypted: MaybeTombstoneRecord = encrypted.decrypt(&keybundle).unwrap(); + assert!(!decrypted.is_tombstone()); + assert_eq!(decrypted, orig_record); + } +}