diff --git a/.github/workflows/tests_and_checks.yml b/.github/workflows/tests_and_checks.yml index b728ad7..8acc4f8 100644 --- a/.github/workflows/tests_and_checks.yml +++ b/.github/workflows/tests_and_checks.yml @@ -21,7 +21,7 @@ jobs: - stable - nightly # minimum version - - 1.64 + - 1.66 steps: - name: Checkout Repository uses: actions/checkout@v3 diff --git a/Cargo.lock b/Cargo.lock index 4747b69..32f2802 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,36 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -14,6 +44,175 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" +dependencies = [ + "async-lock", + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite", + "log", + "parking", + "polling", + "rustix", + "slab", + "socket2", + "waker-fn", +] + +[[package]] +name = "async-lock" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-once-cell" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b49bd4c5b769125ea6323601c39815848972880efd33ffb2d01f9f909adc699" + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-attributes", + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "async-task" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" + +[[package]] +name = "async-trait" +version = "0.1.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "atomic-waker" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" + [[package]] name = "atty" version = "0.2.14" @@ -31,6 +230,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "bit-set" version = "0.5.3" @@ -52,34 +272,142 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "serde", + "tap", + "wyz", +] + +[[package]] +name = "blake2b_simd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq 0.2.6", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq 0.2.6", +] + +[[package]] +name = "blake3" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" +dependencies = [ + "async-channel", + "async-lock", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "log", +] + [[package]] name = "bumpalo" version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +[[package]] +name = "bytemuck" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +dependencies = [ + "serde", +] + [[package]] name = "car-mirror" version = "0.1.0" dependencies = [ "anyhow", + "async-std", + "async-stream", + "async-trait", + "bytes", + "car-mirror", + "deterministic-bloom", + "futures", + "iroh-car", + "libipld", + "libipld-core", "proptest", + "roaring-graphs", + "serde", + "serde_ipld_dagcbor", + "test-strategy", "tracing", "tracing-subscriber", + "wnfs-common", ] [[package]] name = "car-mirror-benches" version = "0.1.0" dependencies = [ + "anyhow", + "async-std", + "async-trait", + "bytes", "car-mirror", "criterion", + "libipld", + "wnfs-common", ] [[package]] @@ -101,6 +429,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + [[package]] name = "cc" version = "1.0.79" @@ -113,6 +450,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "winapi", +] + [[package]] name = "ciborium" version = "0.2.1" @@ -140,6 +489,20 @@ dependencies = [ "half", ] +[[package]] +name = "cid" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd94671561e36e4e7de75f753f577edafb0e7c05d6e4547229fdf7938fbcd2c3" +dependencies = [ + "core2", + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint", +] + [[package]] name = "clap" version = "3.2.25" @@ -161,6 +524,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "concurrent-queue" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -171,6 +543,48 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "constant_time_eq" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" + +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cov-mark" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ffa3d3e0138386cd4361f63537765cac7ee40698028844635a54495a92f67f3" + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.4.0" @@ -205,6 +619,76 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" + +[[package]] +name = "data-encoding-macro" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c904b33cc60130e1aeea4956ab803d08a3f4a0ca82d64ed757afac3891f2bb99" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fdf3fce3ce863539ec1d7fd1b6dcc3c645663376b43ed376bbf887733e4f772" +dependencies = [ + "data-encoding", + "syn 1.0.109", +] + +[[package]] +name = "deterministic-bloom" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12a3873e91e360aee2403cbafd2beb42f02ace06da9b053574518f003aa2490d" +dependencies = [ + "bitvec", + "miette", + "rand_core", + "serde", + "thiserror", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.8.1" @@ -232,6 +716,12 @@ dependencies = [ "libc", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "examples" version = "0.1.0" @@ -254,6 +744,126 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -265,6 +875,24 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "half" version = "1.8.2" @@ -292,6 +920,29 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -322,6 +973,21 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "iroh-car" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a291220adb48738bdea587156c5f44ca5ec4ad31fdeb8fb88fda1dcd7886a24" +dependencies = [ + "anyhow", + "cid", + "futures", + "libipld", + "thiserror", + "tokio", + "unsigned-varint", +] + [[package]] name = "itertools" version = "0.10.5" @@ -346,6 +1012,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -356,25 +1040,200 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" name = "libc" version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" + +[[package]] +name = "libipld" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ccd6b8ffb3afee7081fcaec00e1b099fd1c7ccf35ba5729d88538fcc3b4599" +dependencies = [ + "fnv", + "libipld-cbor", + "libipld-cbor-derive", + "libipld-core", + "libipld-json", + "libipld-macro", + "libipld-pb", + "log", + "multihash", + "thiserror", +] + +[[package]] +name = "libipld-cbor" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d98c9d1747aa5eef1cf099cd648c3fd2d235249f5fed07522aaebc348e423b" +dependencies = [ + "byteorder", + "libipld-core", + "thiserror", +] + +[[package]] +name = "libipld-cbor-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5ba3a729b72973e456a1812b0afe2e176a376c1836cc1528e9fc98ae8cb838" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "libipld-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5acd707e8d8b092e967b2af978ed84709eaded82b75effe6cb6f6cc797ef8158" +dependencies = [ + "anyhow", + "cid", + "core2", + "multibase", + "multihash", + "serde", + "thiserror", +] + +[[package]] +name = "libipld-json" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25856def940047b07b25c33d4e66d248597049ab0202085215dc4dca0487731c" +dependencies = [ + "libipld-core", + "multihash", + "serde", + "serde_json", +] + +[[package]] +name = "libipld-macro" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71171c54214f866ae6722f3027f81dff0931e600e5a61e6b1b6a49ca0b5ed4ae" +dependencies = [ + "libipld-core", +] + +[[package]] +name = "libipld-pb" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f2d0f866c4cd5dc9aa8068c429ba478d2882a3a4b70ab56f7e9a0eddf5d16f" +dependencies = [ + "bytes", + "libipld-core", + "quick-protobuf", + "thiserror", +] + +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +dependencies = [ + "value-bag", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] [[package]] -name = "libm" -version = "0.2.7" +name = "multibase" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] [[package]] -name = "linux-raw-sys" -version = "0.3.8" +name = "multihash" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "cfd8a792c1694c6da4f68db0a9d707c72bd260994da179e6030a5dcee00bb815" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest", + "multihash-derive", + "serde", + "serde-big-array", + "sha2", + "sha3", + "unsigned-varint", +] [[package]] -name = "log" -version = "0.4.19" +name = "multihash-derive" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "1d6d4752e6230d8ef7adf7bd5d8c4b1f6561c1014c5ba9a37445ccefe18aa1db" +dependencies = [ + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] [[package]] name = "nu-ansi-term" @@ -396,6 +1255,15 @@ dependencies = [ "libm", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -420,23 +1288,85 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" + [[package]] name = "pin-project-lite" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys", +] + [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -467,15 +1397,30 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + [[package]] name = "quote" -version = "1.0.28" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -545,6 +1490,41 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +[[package]] +name = "retain_mut" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086" + +[[package]] +name = "roaring" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6106b5cf8587f5834158895e9715a3c6c9716c8aefab57f1f7680917191c7873" +dependencies = [ + "bytemuck", + "byteorder", + "retain_mut", +] + +[[package]] +name = "roaring-graphs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3b6db6a957b3ee92cf83d4a107d37827c4aa7a92ca71a933f0bea83a35d61f" +dependencies = [ + "cov-mark", + "proptest", + "rand", + "roaring", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustix" version = "0.37.20" @@ -592,24 +1572,60 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" -version = "1.0.164" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd31f59f6fe2b0c055371bb2f16d7f0aa7d8881676c04a55b1596d1a17cd10a4" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace39c1b7526be78c755a4c698313f699cf44e62408c0029bf9ab9450fe836da" +dependencies = [ + "cbor4ii", + "cid", + "scopeguard", + "serde", ] [[package]] @@ -623,6 +1639,27 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -632,23 +1669,94 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "structmeta" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.28", +] + +[[package]] +name = "structmeta-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" -version = "2.0.18" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.6.0" @@ -663,12 +1771,44 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "test-strategy" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8361c808554228ad09bfed70f5c823caf8a3450b6881cc3a38eb57e8c08c1d9" +dependencies = [ + "proc-macro2", + "quote", + "structmeta", + "syn 2.0.28", +] + [[package]] name = "textwrap" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "thread_local" version = "1.1.7" @@ -689,6 +1829,27 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio" +version = "1.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +dependencies = [ + "autocfg", + "backtrace", + "bytes", + "pin-project-lite", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "tracing" version = "0.1.37" @@ -709,7 +1870,7 @@ checksum = "8803eee176538f94ae9a14b55b2804eb7e1441f8210b1c31290b3bccdccff73b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] @@ -747,6 +1908,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + [[package]] name = "unarray" version = "0.1.4" @@ -759,12 +1926,42 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unsigned-varint" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" + [[package]] name = "valuable" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -774,6 +1971,12 @@ dependencies = [ "libc", ] +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "walkdir" version = "2.3.3" @@ -813,7 +2016,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -847,7 +2050,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -923,6 +2126,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -988,3 +2200,38 @@ name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "wnfs-common" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfcb4584f3866ead49adae8c05cec6f633139d19283448aa7807280612e24b7" +dependencies = [ + "anyhow", + "async-once-cell", + "async-trait", + "bytes", + "chrono", + "futures", + "libipld", + "multihash", + "once_cell", + "rand_core", + "serde", + "thiserror", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" diff --git a/Cargo.toml b/Cargo.toml index 408bb4f..aa734b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [workspace] members = [ - "examples", "car-mirror", "car-mirror-benches", "car-mirror-wasm" -] +, + "examples"] # See https://doc.rust-lang.org/cargo/reference/profiles.html for more info. [profile.release.package.car-mirror-wasm] @@ -23,3 +23,4 @@ opt-level = "s" # or 'z' to optimize "aggressively" for size # See https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#splitting-debug-information [profile.dev] split-debuginfo = "unpacked" +opt-level = 3 diff --git a/car-mirror-benches/Cargo.toml b/car-mirror-benches/Cargo.toml index 4d5161e..ba1c467 100644 --- a/car-mirror-benches/Cargo.toml +++ b/car-mirror-benches/Cargo.toml @@ -6,11 +6,21 @@ edition = "2021" authors = ["Stephen Akinyemi "] [dependencies] +anyhow = "1.0" +async-std = { version = "1.11", features = ["attributes"] } +async-trait = "0.1" +bytes = "1.4.0" car-mirror = { path = "../car-mirror", version = "0.1", features = ["test_utils"] } +libipld = "0.16.0" +wnfs-common = "0.1.23" [dev-dependencies] criterion = { version = "0.4", default-features = false } [[bench]] -name = "a_benchmark" +name = "in_memory" +harness = false + +[[bench]] +name = "artificially_slow_blockstore" harness = false diff --git a/car-mirror-benches/benches/a_benchmark.rs b/car-mirror-benches/benches/a_benchmark.rs deleted file mode 100644 index 6650d1a..0000000 --- a/car-mirror-benches/benches/a_benchmark.rs +++ /dev/null @@ -1,15 +0,0 @@ -use criterion::{criterion_group, criterion_main, Criterion}; - -pub fn add_benchmark(c: &mut Criterion) { - let mut rvg = car_mirror::test_utils::Rvg::deterministic(); - let int_val_1 = rvg.sample(&(0..100i32)); - let int_val_2 = rvg.sample(&(0..100i32)); - - c.bench_function("add", |b| { - b.iter(|| { - car_mirror::add(int_val_1, int_val_2); - }) - }); -} -criterion_group!(benches, add_benchmark); -criterion_main!(benches); diff --git a/car-mirror-benches/benches/artificially_slow_blockstore.rs b/car-mirror-benches/benches/artificially_slow_blockstore.rs new file mode 100644 index 0000000..f74ec72 --- /dev/null +++ b/car-mirror-benches/benches/artificially_slow_blockstore.rs @@ -0,0 +1,117 @@ +use anyhow::Result; +use async_trait::async_trait; +use bytes::Bytes; +use car_mirror::{ + common::Config, + pull, push, + test_utils::{arb_ipld_dag, links_to_padded_ipld, setup_blockstore}, +}; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use libipld::Cid; +use std::time::Duration; +use wnfs_common::{BlockStore, MemoryBlockStore}; + +pub fn push_throttled(c: &mut Criterion) { + let mut rvg = car_mirror::test_utils::Rvg::deterministic(); + + c.bench_function("push cold, get_block throttled", |b| { + b.iter_batched( + || { + let (blocks, root) = rvg.sample(&arb_ipld_dag( + 60..64, + 0.9, // Very highly connected + links_to_padded_ipld(10 * 1024), + )); + let store = async_std::task::block_on(setup_blockstore(blocks)).unwrap(); + (store, root) + }, + |(client_store, root)| { + let client_store = &ThrottledBlockStore(client_store); + let server_store = &ThrottledBlockStore::new(); + let config = &Config::default(); + + // Simulate a multi-round protocol run in-memory + async_std::task::block_on(async move { + let mut request = push::request(root, None, config, client_store).await?; + loop { + let response = push::response(root, request, config, server_store).await?; + + if response.indicates_finished() { + break; + } + request = push::request(root, Some(response), config, client_store).await?; + } + + Ok::<(), anyhow::Error>(()) + }) + .unwrap(); + }, + BatchSize::LargeInput, + ) + }); +} + +pub fn pull_throttled(c: &mut Criterion) { + let mut rvg = car_mirror::test_utils::Rvg::deterministic(); + + c.bench_function("pull cold, get_block throttled", |b| { + b.iter_batched( + || { + let (blocks, root) = rvg.sample(&arb_ipld_dag( + 60..64, + 0.9, // Very highly connected + links_to_padded_ipld(10 * 1024), // 10KiB random data added + )); + let store = async_std::task::block_on(setup_blockstore(blocks)).unwrap(); + (store, root) + }, + |(server_store, root)| { + let server_store = &ThrottledBlockStore(server_store); + let client_store = &ThrottledBlockStore::new(); + let config = &Config::default(); + + // Simulate a multi-round protocol run in-memory + async_std::task::block_on(async move { + let mut request = pull::request(root, None, config, client_store).await?; + loop { + let response = pull::response(root, request, config, server_store).await?; + request = pull::request(root, Some(response), config, client_store).await?; + + if request.indicates_finished() { + break; + } + } + + Ok::<(), anyhow::Error>(()) + }) + .unwrap(); + }, + BatchSize::LargeInput, + ) + }); +} + +#[derive(Debug, Clone)] +struct ThrottledBlockStore(MemoryBlockStore); + +#[async_trait(?Send)] +impl BlockStore for ThrottledBlockStore { + async fn get_block(&self, cid: &Cid) -> Result { + let bytes = self.0.get_block(cid).await?; + async_std::task::sleep(Duration::from_micros(50)).await; // Block fetching is artifically slowed by 50 microseconds + Ok(bytes) + } + + async fn put_block(&self, bytes: impl Into, codec: u64) -> Result { + self.0.put_block(bytes, codec).await + } +} + +impl ThrottledBlockStore { + pub fn new() -> Self { + Self(MemoryBlockStore::new()) + } +} + +criterion_group!(benches, push_throttled, pull_throttled); +criterion_main!(benches); diff --git a/car-mirror-benches/benches/in_memory.rs b/car-mirror-benches/benches/in_memory.rs new file mode 100644 index 0000000..84b4291 --- /dev/null +++ b/car-mirror-benches/benches/in_memory.rs @@ -0,0 +1,88 @@ +use car_mirror::{ + common::Config, + pull, push, + test_utils::{arb_ipld_dag, links_to_padded_ipld, setup_blockstore}, +}; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use wnfs_common::MemoryBlockStore; + +pub fn push(c: &mut Criterion) { + let mut rvg = car_mirror::test_utils::Rvg::deterministic(); + + c.bench_function("push cold", |b| { + b.iter_batched( + || { + let (blocks, root) = rvg.sample(&arb_ipld_dag( + 250..256, + 0.9, // Very highly connected + links_to_padded_ipld(10 * 1024), + )); + let store = async_std::task::block_on(setup_blockstore(blocks)).unwrap(); + (store, root) + }, + |(ref client_store, root)| { + let server_store = &MemoryBlockStore::new(); + let config = &Config::default(); + + // Simulate a multi-round protocol run in-memory + async_std::task::block_on(async move { + let mut request = push::request(root, None, config, client_store).await?; + loop { + let response = push::response(root, request, config, server_store).await?; + + if response.indicates_finished() { + break; + } + request = push::request(root, Some(response), config, client_store).await?; + } + + Ok::<(), anyhow::Error>(()) + }) + .unwrap(); + }, + BatchSize::LargeInput, + ) + }); +} + +pub fn pull(c: &mut Criterion) { + let mut rvg = car_mirror::test_utils::Rvg::deterministic(); + + c.bench_function("pull cold", |b| { + b.iter_batched( + || { + let (blocks, root) = rvg.sample(&arb_ipld_dag( + 250..256, + 0.9, // Very highly connected + links_to_padded_ipld(10 * 1024), // 10KiB random data per block + )); + let store = async_std::task::block_on(setup_blockstore(blocks)).unwrap(); + (store, root) + }, + |(ref server_store, root)| { + let client_store = &MemoryBlockStore::new(); + let config = &Config::default(); + + // Simulate a multi-round protocol run in-memory + async_std::task::block_on(async move { + let mut request = pull::request(root, None, config, client_store).await?; + loop { + let response = pull::response(root, request, config, server_store).await?; + request = pull::request(root, Some(response), config, client_store).await?; + + if request.indicates_finished() { + break; + } + } + + Ok::<(), anyhow::Error>(()) + }) + .unwrap(); + }, + BatchSize::LargeInput, + ) + }); +} + +criterion_group!(benches, push, pull); +criterion_main!(benches); diff --git a/car-mirror-wasm/Cargo.toml b/car-mirror-wasm/Cargo.toml index 97e75a3..0dcdd3a 100644 --- a/car-mirror-wasm/Cargo.toml +++ b/car-mirror-wasm/Cargo.toml @@ -8,7 +8,7 @@ include = ["/src", "README.md", "LICENSE-APACHE", "LICENSE-MIT"] license = "Apache-2.0 or MIT" readme = "README.md" edition = "2021" -rust-version = "1.64" +rust-version = "1.66" documentation = "https://docs.rs/car-mirror-wasm" repository = "https://github.com/fission-codes/rs-car-mirror/tree/main/car-mirror-wasm" authors = ["Stephen Akinyemi "] diff --git a/car-mirror/Cargo.toml b/car-mirror/Cargo.toml index 7a95196..3a59ad5 100644 --- a/car-mirror/Cargo.toml +++ b/car-mirror/Cargo.toml @@ -8,7 +8,7 @@ include = ["/src", "README.md", "LICENSE-APACHE", "LICENSE-MIT"] license = "Apache-2.0 or MIT" readme = "README.md" edition = "2021" -rust-version = "1.64" +rust-version = "1.66" documentation = "https://docs.rs/car-mirror" repository = "https://github.com/fission-codes/rs-car-mirror/tree/main/car-mirror" authors = ["Stephen Akinyemi "] @@ -24,16 +24,32 @@ doc = true [dependencies] anyhow = "1.0" +async-stream = "0.3.5" +async-trait = "0.1.73" +bytes = "1.4.0" +deterministic-bloom = "0.1" +futures = "0.3.28" +iroh-car = "0.3.0" +libipld = "0.16.0" +libipld-core = "0.16.0" proptest = { version = "1.1", optional = true } +roaring-graphs = { version = "0.12", optional = true } +serde = "1.0.183" +serde_ipld_dagcbor = "0.4.0" tracing = "0.1" tracing-subscriber = "0.3" +wnfs-common = "0.1.23" [dev-dependencies] +async-std = { version = "1.11", features = ["attributes"] } +car-mirror = { path = ".", features = ["test_utils"] } proptest = "1.1" +roaring-graphs = "0.12" +test-strategy = "0.3" [features] default = [] -test_utils = ["proptest"] +test_utils = ["proptest", "roaring-graphs"] [package.metadata.docs.rs] all-features = true diff --git a/car-mirror/proptest-regressions/lib.txt b/car-mirror/proptest-regressions/lib.txt new file mode 100644 index 0000000..b3f356f --- /dev/null +++ b/car-mirror/proptest-regressions/lib.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ecfcb732e093de600a3a3012674efabacf88f817b47d75c0ad0da9762ca3b6f7 # shrinks to input = _WalkDagNeverIteratesBlockTwiceArgs { dag: [(Cid(bafyreigvk2vd4s7ecxqhr7vlf5ei5tpdpalx73wbt53zokomvab5ouzq2a), b"\x86\xd8*X%\0\x01q\x12 pg\xacO\xb3\xfe\xacy\xd9k\xee\xc10\xdd\x8b\xbbc\x81\xc1\x06\x12\xf5Uw\x9e\xed\x11n\r?8\xdc\xd8*X%\0\x01q\x12 \xe0\xed\xdb\xa6\x0c\x8b\xf9\xe2fk\x12\xdd3\xf9\xb9Y%[\x85\xf6\xd9\xd8\x15\x8a\xce3\xbb\xdfN\xcfM\x93\xd8*X%\0\x01q\x12 \xfei\x9fJr\x7f\xed\xfd\t\0\x02lz5\x0eD\xc5\xf9\xe2\xda\"\x9ez5y\xd5\xc8\x02\xab+\xc0\x92\xd8*X%\0\x01q\x12 \xc6M\xa8\xd1B\x12\xcfT\xbfC\xd0\x1e\x89\x8c\xaa\x11\xafq\xed^sF\xb5\xda\x19\x98\xf2B\xb9\xe2\x9f\xb3\xd8*X%\0\x01q\x12 v\xbe\x8bR\x8d\0u\xf7\xaa\xe9\x8do\xa5zm<\x83\xaeH\n\x84i\xe6h\xd7\xb0\xaf\x96\x89\x95\xacq\xd8*X%\0\x01q\x12 \x92`zHK\xcfR\xb8\x10G\xa3+t\t\xb7_\xc9\xf6\x9b+\xee\xe0\x83S\"#\xba\xe8Q\xdd\x10\x02")] } diff --git a/car-mirror/src/common.rs b/car-mirror/src/common.rs new file mode 100644 index 0000000..900aa32 --- /dev/null +++ b/car-mirror/src/common.rs @@ -0,0 +1,316 @@ +use anyhow::{anyhow, bail, Result}; +use bytes::Bytes; +use deterministic_bloom::runtime_size::BloomFilter; +use futures::TryStreamExt; +use iroh_car::{CarHeader, CarReader, CarWriter}; +use libipld::{Ipld, IpldCodec}; +use libipld_core::{cid::Cid, codec::References}; +use std::io::Cursor; +use wnfs_common::BlockStore; + +use crate::{ + dag_walk::DagWalk, + incremental_verification::{BlockState, IncrementalDagVerification}, + messages::{Bloom, PullRequest, PushResponse}, +}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Configuration values (such as byte limits) for the CAR mirror protocol +#[derive(Clone, Debug)] +pub struct Config { + /// A client will try to send at least `send_minimum` bytes of block data + /// in each request, except if close to the end of the protocol (when there's) + /// not that much data left. + pub send_minimum: usize, + /// The maximum number of bytes per request that the server accepts. + pub receive_maximum: usize, + /// The maximum number of roots per request that the server will send to the client, + /// and that the client will consume. + pub max_roots_per_round: usize, + /// The target false positive rate for the bloom filter that the server sends. + pub bloom_fpr: fn(u64) -> f64, +} + +/// Some information that the block receiving end provides the block sending end +/// in order to deduplicate block transfers. +#[derive(Debug, Clone)] +pub struct ReceiverState { + /// At least *some* of the subgraph roots that are missing for sure on the receiving end. + pub missing_subgraph_roots: Vec, + /// An optional bloom filter of all CIDs below the root that the receiving end has. + pub have_cids_bloom: Option, +} + +/// Newtype around bytes that are supposed to represent a CAR file +#[derive(Debug, Clone)] +pub struct CarFile { + /// The car file contents as bytes. + /// (`CarFile` is cheap to clone, since `Bytes` is an `Arc` wrapper around a byte buffer.) + pub bytes: Bytes, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// This function is run on the block sending side of the protocol. +/// +/// It's used on the client during the push protocol, or on the server +/// during the pull protocol. +/// +/// It returns a `CarFile` of (a subset) of all blocks below `root`, that +/// are thought to be missing on the receiving end. +pub async fn block_send( + root: Cid, + last_state: Option, + config: &Config, + store: &impl BlockStore, +) -> Result { + let ReceiverState { + ref missing_subgraph_roots, + have_cids_bloom, + } = last_state.unwrap_or(ReceiverState { + missing_subgraph_roots: vec![root], + have_cids_bloom: None, + }); + + // Verify that all missing subgraph roots are in the relevant DAG: + let subgraph_roots: Vec = DagWalk::breadth_first([root]) + .stream(store) + .try_filter_map(|(cid, _)| async move { + Ok(missing_subgraph_roots.contains(&cid).then_some(cid)) + }) + .try_collect() + .await?; + + let bloom = have_cids_bloom.unwrap_or_else(|| BloomFilter::new_with(1, Box::new([0]))); // An empty bloom that contains nothing + + let mut writer = CarWriter::new( + CarHeader::new_v1( + // https://github.com/wnfs-wg/car-mirror-spec/issues/6 + // CAR files *must* have at least one CID in them, and all of them + // need to appear as a block in the payload. + // It would probably make most sense to just write all subgraph roots into this, + // but we don't know how many of the subgraph roots fit into this round yet, + // so we're simply writing the first one in here, since we know + // at least one block will be written (and it'll be that one). + subgraph_roots.iter().take(1).cloned().collect(), + ), + Vec::new(), + ); + + writer.write_header().await?; + + let mut block_bytes = 0; + let mut dag_walk = DagWalk::breadth_first(subgraph_roots.clone()); + while let Some((cid, block)) = dag_walk.next(store).await? { + if bloom.contains(&cid.to_bytes()) && !subgraph_roots.contains(&cid) { + continue; + } + + writer.write(cid, &block).await?; + + // TODO(matheus23): Count the actual bytes sent? + // At the moment, this is a rough estimate. iroh-car could be improved to return the written bytes. + block_bytes += block.len(); + if block_bytes > config.send_minimum { + break; + } + } + + Ok(CarFile { + bytes: writer.finish().await?.into(), + }) +} + +/// This function is run on the block receiving end of the protocol. +/// +/// It's used on the client during the pull protocol and on the server +/// during the push protocol. +/// +/// It takes a `CarFile`, verifies that its contents are related to the +/// `root` and returns some information to help the block sending side +/// figure out what blocks to send next. +pub async fn block_receive( + root: Cid, + last_car: Option, + config: &Config, + store: &impl BlockStore, +) -> Result { + let mut dag_verification = IncrementalDagVerification::new([root], store).await?; + + if let Some(car) = last_car { + let mut reader = CarReader::new(Cursor::new(car.bytes)).await?; + let mut block_bytes = 0; + + while let Some((cid, vec)) = reader.next_block().await? { + let block = Bytes::from(vec); + + block_bytes += block.len(); + if block_bytes > config.receive_maximum { + bail!( + "Received more than {} bytes ({block_bytes}), aborting request.", + config.receive_maximum + ); + } + + match dag_verification.block_state(cid) { + BlockState::Have => continue, + BlockState::Unexpected => { + eprintln!("Warn: Received block {cid} out of order, may be due to bloom false positive."); + break; + } + BlockState::Want => { + dag_verification + .verify_and_store_block((cid, block), store) + .await?; + } + } + } + } + + let missing_subgraph_roots = dag_verification + .want_cids + .iter() + .take(config.max_roots_per_round) + .cloned() + .collect(); + + let bloom_capacity = dag_verification.have_cids.len() as u64; + + if bloom_capacity == 0 { + return Ok(ReceiverState { + missing_subgraph_roots, + have_cids_bloom: None, + }); + } + + let mut bloom = + BloomFilter::new_from_fpr_po2(bloom_capacity, (config.bloom_fpr)(bloom_capacity)); + + dag_verification + .have_cids + .iter() + .for_each(|cid| bloom.insert(&cid.to_bytes())); + + Ok(ReceiverState { + missing_subgraph_roots, + have_cids_bloom: Some(bloom), + }) +} + +/// Find all CIDs that a block references. +/// +/// This will error out if +/// - the codec is not supported +/// - the block can't be parsed. +pub fn references>(cid: Cid, block: impl AsRef<[u8]>, mut refs: E) -> Result { + let codec: IpldCodec = cid + .codec() + .try_into() + .map_err(|_| anyhow!("Unsupported codec in Cid: {cid}"))?; + + >::references(codec, &mut Cursor::new(block), &mut refs)?; + Ok(refs) +} + +//-------------------------------------------------------------------------------------------------- +// Implementations +//-------------------------------------------------------------------------------------------------- + +impl From for ReceiverState { + fn from(push: PushResponse) -> Self { + let PushResponse { + subgraph_roots, + bloom, + } = push; + + Self { + missing_subgraph_roots: subgraph_roots, + have_cids_bloom: Self::bloom_deserialize(bloom), + } + } +} + +impl From for ReceiverState { + fn from(pull: PullRequest) -> Self { + let PullRequest { resources, bloom } = pull; + + Self { + missing_subgraph_roots: resources, + have_cids_bloom: Self::bloom_deserialize(bloom), + } + } +} + +impl From for PushResponse { + fn from(receiver_state: ReceiverState) -> PushResponse { + let ReceiverState { + missing_subgraph_roots, + have_cids_bloom, + } = receiver_state; + + let bloom = ReceiverState::bloom_serialize(have_cids_bloom); + + PushResponse { + subgraph_roots: missing_subgraph_roots, + bloom, + } + } +} + +impl From for PullRequest { + fn from(receiver_state: ReceiverState) -> PullRequest { + let ReceiverState { + missing_subgraph_roots, + have_cids_bloom, + } = receiver_state; + + let bloom = ReceiverState::bloom_serialize(have_cids_bloom); + + PullRequest { + resources: missing_subgraph_roots, + bloom, + } + } +} + +impl ReceiverState { + fn bloom_serialize(bloom: Option) -> Bloom { + match bloom { + Some(bloom) => Bloom { + hash_count: bloom.hash_count() as u32, + bytes: bloom.as_bytes().to_vec(), + }, + None => Bloom { + hash_count: 3, + bytes: Vec::new(), + }, + } + } + + fn bloom_deserialize(bloom: Bloom) -> Option { + if bloom.bytes.is_empty() { + None + } else { + Some(BloomFilter::new_with( + bloom.hash_count as usize, + bloom.bytes.into_boxed_slice(), + )) + } + } +} + +impl Default for Config { + fn default() -> Self { + Self { + send_minimum: 128 * 1024, // 128KiB + receive_maximum: 512 * 1024, // 512KiB + max_roots_per_round: 1000, // max. ~41KB of CIDs + bloom_fpr: |num_of_elems| 0.1 / num_of_elems as f64, + } + } +} diff --git a/car-mirror/src/dag_walk.rs b/car-mirror/src/dag_walk.rs new file mode 100644 index 0000000..3f27e7e --- /dev/null +++ b/car-mirror/src/dag_walk.rs @@ -0,0 +1,227 @@ +use crate::common::references; +use anyhow::Result; +use bytes::Bytes; +use futures::{stream::try_unfold, Stream}; +use libipld_core::cid::Cid; +use std::collections::{HashSet, VecDeque}; +use wnfs_common::BlockStore; + +/// A struct that represents an ongoing walk through the Dag. +#[derive(Clone, Debug)] +pub struct DagWalk { + /// A queue of CIDs to visit next + pub frontier: VecDeque, + /// The set of already visited CIDs. This prevents re-visiting. + pub visited: HashSet, + /// Whether to do a breadth-first or depth-first traversal. + /// This controls whether newly discovered links are appended or prepended to the frontier. + pub breadth_first: bool, +} + +impl DagWalk { + /// Start a breadth-first traversal of given roots. + /// + /// Breadth-first is explained the easiest in the simple case of a tree (which is a DAG): + /// It will visit each node in the tree layer-by-layer. + /// + /// So the first nodes it will visit are going to be all roots in order. + pub fn breadth_first(roots: impl IntoIterator) -> Self { + Self::new(roots, true) + } + + /// Start a depth-first traversal of given roots. + /// + /// Depth-first will follow links immediately after discovering them, taking the fastest + /// path towards leaves. + /// + /// The very first node is guaranteed to be the first root, but subsequent nodes may not be + /// from the initial roots. + pub fn depth_first(roots: impl IntoIterator) -> Self { + Self::new(roots, false) + } + + /// Start a DAG traversal of given roots. See also `breadth_first` and `depth_first`. + pub fn new(roots: impl IntoIterator, breadth_first: bool) -> Self { + let frontier = roots.into_iter().collect(); + let visited = HashSet::new(); + Self { + frontier, + visited, + breadth_first, + } + } + + /// Return the next node in the traversal. + /// + /// Returns `None` if no nodes are left to be visited. + pub async fn next(&mut self, store: &impl BlockStore) -> Result> { + let cid = loop { + let popped = if self.breadth_first { + self.frontier.pop_back() + } else { + self.frontier.pop_front() + }; + + let Some(cid) = popped else { + return Ok(None); + }; + + // We loop until we find an unvisited block + if self.visited.insert(cid) { + break cid; + } + }; + + // TODO: Two opportunities for performance improvement: + // - skip Raw CIDs. They can't have further links (but needs adjustment to this function's return type) + // - run multiple `get_block` calls concurrently + let block = store.get_block(&cid).await?; + for ref_cid in references(cid, &block, Vec::new())? { + if !self.visited.contains(&ref_cid) { + self.frontier.push_front(ref_cid); + } + } + + Ok(Some((cid, block))) + } + + /// Turn this traversal into a stream + pub fn stream( + self, + store: &impl BlockStore, + ) -> impl Stream> + Unpin + '_ { + Box::pin(try_unfold(self, move |mut this| async move { + let maybe_block = this.next(store).await?; + Ok(maybe_block.map(|b| (b, this))) + })) + } + + /// Find out whether the traversal is finished. + /// + /// The next call to `next` would result in `None` if this returns true. + pub fn is_finished(&self) -> bool { + // We're finished if the frontier does not contain any CIDs that we have not visited yet. + // Put differently: + // We're not finished if there exist unvisited CIDs in the frontier. + !self + .frontier + .iter() + .any(|frontier_cid| !self.visited.contains(frontier_cid)) + } + + /// Skip a node from the traversal for now. + pub fn skip_walking(&mut self, block: (Cid, Bytes)) -> Result<()> { + let (cid, bytes) = block; + let refs = references(cid, bytes, HashSet::new())?; + self.visited.insert(cid); + self.frontier + .retain(|frontier_cid| !refs.contains(frontier_cid)); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::TryStreamExt; + use libipld::Ipld; + use wnfs_common::MemoryBlockStore; + + #[async_std::test] + async fn test_walk_dag_breadth_first() -> Result<()> { + let store = &MemoryBlockStore::new(); + + // cid_root ---> cid_1_wrap ---> cid_1 + // -> cid_2 + // -> cid_3 + + let cid_1 = store.put_serializable(&Ipld::String("1".into())).await?; + let cid_2 = store.put_serializable(&Ipld::String("2".into())).await?; + let cid_3 = store.put_serializable(&Ipld::String("3".into())).await?; + + let cid_1_wrap = store + .put_serializable(&Ipld::List(vec![Ipld::Link(cid_1)])) + .await?; + + let cid_root = store + .put_serializable(&Ipld::List(vec![ + Ipld::Link(cid_1_wrap), + Ipld::Link(cid_2), + Ipld::Link(cid_3), + ])) + .await?; + + let cids = DagWalk::breadth_first([cid_root]) + .stream(store) + .try_collect::>() + .await? + .into_iter() + .map(|(cid, _block)| cid) + .collect::>(); + + assert_eq!(cids, vec![cid_root, cid_1_wrap, cid_2, cid_3, cid_1]); + + Ok(()) + } +} + +#[cfg(test)] +mod proptests { + use super::*; + use crate::test_utils::arb_ipld_dag; + use futures::TryStreamExt; + use libipld::{ + multihash::{Code, MultihashDigest}, + Cid, Ipld, IpldCodec, + }; + use proptest::strategy::Strategy; + use std::collections::BTreeSet; + use test_strategy::proptest; + use wnfs_common::{dagcbor::encode, BlockStore, MemoryBlockStore}; + + fn ipld_dags() -> impl Strategy, Cid)> { + arb_ipld_dag(1..256, 0.5, |cids, _| { + let ipld = Ipld::List(cids.into_iter().map(Ipld::Link).collect()); + let cid = Cid::new_v1( + IpldCodec::DagCbor.into(), + Code::Blake3_256.digest(&encode(&ipld).unwrap()), + ); + (cid, ipld) + }) + } + + #[proptest(max_shrink_iters = 100_000)] + fn walk_dag_never_iterates_block_twice(#[strategy(ipld_dags())] dag: (Vec<(Cid, Ipld)>, Cid)) { + async_std::task::block_on(async { + let (dag, root) = dag; + let store = &MemoryBlockStore::new(); + for (cid, ipld) in dag.iter() { + let block: Bytes = encode(ipld).unwrap().into(); + let cid_store = store + .put_block(block, IpldCodec::DagCbor.into()) + .await + .unwrap(); + assert_eq!(*cid, cid_store); + } + + let mut cids = DagWalk::breadth_first([root]) + .stream(store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await + .unwrap(); + + cids.sort(); + + let unique_cids = cids + .iter() + .cloned() + .collect::>() + .into_iter() + .collect::>(); + + assert_eq!(cids, unique_cids); + }); + } +} diff --git a/car-mirror/src/incremental_verification.rs b/car-mirror/src/incremental_verification.rs new file mode 100644 index 0000000..0960699 --- /dev/null +++ b/car-mirror/src/incremental_verification.rs @@ -0,0 +1,140 @@ +use crate::dag_walk::DagWalk; +use anyhow::{anyhow, bail, Result}; +use bytes::Bytes; +use libipld_core::{ + cid::Cid, + multihash::{Code, MultihashDigest}, +}; +use std::{collections::HashSet, matches}; +use wnfs_common::{BlockStore, BlockStoreError}; + +/// A data structure that keeps state about incremental DAG verification. +#[derive(Clone, Debug)] +pub struct IncrementalDagVerification { + /// All the CIDs that have been discovered to be missing from the DAG. + pub want_cids: HashSet, + /// All the CIDs that are available locally. + pub have_cids: HashSet, +} + +/// The state of a block retrieval +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BlockState { + /// The block was already received/is already stored + Have, + /// We know we will need this block + Want, + /// We don't know whether we'll need this block + Unexpected, +} + +impl IncrementalDagVerification { + /// Initiate incremental DAG verification of given roots. + /// + /// This will already run a traversal to find missing subgraphs and + /// CIDs that are already present. + pub async fn new( + roots: impl IntoIterator, + store: &impl BlockStore, + ) -> Result { + let mut this = Self { + want_cids: roots.into_iter().collect(), + have_cids: HashSet::new(), + }; + + this.update_have_cids(store).await?; + + Ok(this) + } + + async fn update_have_cids(&mut self, store: &impl BlockStore) -> Result<()> { + let mut dag_walk = DagWalk::breadth_first(self.want_cids.iter().cloned()); + + loop { + match dag_walk.next(store).await { + Err(e) => { + if let Some(BlockStoreError::CIDNotFound(not_found)) = + e.downcast_ref::() + { + self.want_cids.insert(*not_found); + } else { + bail!(e); + } + } + Ok(Some((cid, _))) => { + self.want_cids.remove(&cid); + self.have_cids.insert(cid); + } + Ok(None) => { + break; + } + } + } + + Ok(()) + } + + /// Check the state of a CID to find out whether + /// - we expect it as one of the next possible blocks to receive (Want) + /// - we have already stored it (Have) + /// - we don't know whether we need it (Unexpected) + pub fn block_state(&self, cid: Cid) -> BlockState { + if self.want_cids.contains(&cid) { + BlockState::Want + } else if self.have_cids.contains(&cid) { + BlockState::Have + } else { + BlockState::Unexpected + } + } + + /// Verify that + /// - the block is part of the graph below the roots. + /// - the block hasn't been received before + /// - the block actually hashes to the hash from given CID and + /// + /// And finally stores the block in the blockstore. + /// + /// This *may* fail, even if the block is part of the graph below the roots, + /// if intermediate blocks between the roots and this block are missing. + /// + /// This *may* add the block to the blockstore, but still fail to verify, specifically + /// if the block's bytes don't match the hash in the CID. + pub async fn verify_and_store_block( + &mut self, + block: (Cid, Bytes), + store: &impl BlockStore, + ) -> Result<()> { + let (cid, bytes) = block; + + let block_state = self.block_state(cid); + if !matches!(block_state, BlockState::Want) { + bail!("Incremental verification failed. Block state is: {block_state:?}, expected BlockState::Want"); + } + + let hash_func: Code = cid + .hash() + .code() + .try_into() + .map_err(|_| anyhow!("Unsupported hash code in CID {cid}"))?; + + let hash = hash_func.digest(bytes.as_ref()); + + if &hash != cid.hash() { + let result_cid = Cid::new_v1(cid.codec(), hash); + bail!("Digest mismatch in CAR file: expected {cid}, got {result_cid}"); + } + + let result_cid = store.put_block(bytes, cid.codec()).await?; + + // TODO(matheus23): The BlockStore chooses the hashing function, + // so it may choose a different hashing function, causing a mismatch + if result_cid != cid { + bail!("BlockStore uses an incompatible hashing function: CID mismatched, expected {cid}, got {result_cid}"); + } + + self.update_have_cids(store).await?; + + Ok(()) + } +} diff --git a/car-mirror/src/lib.rs b/car-mirror/src/lib.rs index 2da8f41..b40d1a0 100644 --- a/car-mirror/src/lib.rs +++ b/car-mirror/src/lib.rs @@ -4,27 +4,20 @@ //! car-mirror -/// Test utilities. +/// Test utilities. Enabled with the `test_utils` feature flag. #[cfg(any(test, feature = "test_utils"))] #[cfg_attr(docsrs, doc(cfg(feature = "test_utils")))] pub mod test_utils; -/// Add two integers together. -pub fn add(a: i32, b: i32) -> i32 { - a + b -} - -/// Multiplies two integers together. -pub fn mult(a: i32, b: i32) -> i32 { - a * b -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_mult() { - assert_eq!(mult(3, 2), 6); - } -} +/// Common utilities +pub mod common; +/// Algorithms for walking IPLD directed acyclic graphs +pub mod dag_walk; +/// Algorithms for doing incremental verification of IPLD DAGs on the receiving end. +pub mod incremental_verification; +/// Data types that are sent over-the-wire and relevant serialization code. +pub mod messages; +/// The CAR mirror pull protocol. Meant to be used qualified, i.e. `pull::request` and `pull::response` +pub mod pull; +/// The CAR mirror push protocol. Meant to be used qualified, i.e. `push::request` and `push::response` +pub mod push; diff --git a/car-mirror/src/messages.rs b/car-mirror/src/messages.rs new file mode 100644 index 0000000..85b1640 --- /dev/null +++ b/car-mirror/src/messages.rs @@ -0,0 +1,60 @@ +use libipld_core::cid::Cid; +use serde::{Deserialize, Serialize}; + +/// Initial message for pull requests. +/// +/// Over-the-wire data type from the [specification]. +/// +/// [specification]: https://github.com/fission-codes/spec/blob/86fcfb07d507f1df4fdaaf49088abecbb1dda76a/car-pool/car-mirror/http.md#12-requestor-payload +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PullRequest { + /// Requested CID roots + #[serde(rename = "rs")] + pub resources: Vec, + + /// A bloom containing already stored blocks + #[serde(flatten)] + pub bloom: Bloom, +} + +/// The response sent after the initial and subsequent push requests. +/// +/// Wire data type from the [specification]. +/// +/// [specification]: https://github.com/fission-codes/spec/blob/86fcfb07d507f1df4fdaaf49088abecbb1dda76a/car-pool/car-mirror/http.md#23-provider-payload +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PushResponse { + /// Incomplete subgraph roots + #[serde(rename = "sr")] + pub subgraph_roots: Vec, + + /// A bloom containing already stored blocks + #[serde(flatten)] + pub bloom: Bloom, +} + +/// The serialization format for bloom filters in CAR mirror +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Bloom { + /// Bloom filter hash count + #[serde(rename = "bk")] + pub hash_count: u32, + + /// Bloom filter Binary + #[serde(rename = "bb")] + pub bytes: Vec, +} + +impl PushResponse { + /// Whether this response indicates that the protocol is finished. + pub fn indicates_finished(&self) -> bool { + self.subgraph_roots.is_empty() + } +} + +impl PullRequest { + /// Whether you need to actually send the request or not. If true, this indicates that the protocol is finished. + pub fn indicates_finished(&self) -> bool { + self.resources.is_empty() + } +} diff --git a/car-mirror/src/pull.rs b/car-mirror/src/pull.rs new file mode 100644 index 0000000..8bfdbb8 --- /dev/null +++ b/car-mirror/src/pull.rs @@ -0,0 +1,153 @@ +use crate::{ + common::{block_receive, block_send, CarFile, Config, ReceiverState}, + messages::PullRequest, +}; +use anyhow::Result; +use libipld::Cid; +use wnfs_common::BlockStore; + +/// Create a CAR mirror pull request. +/// +/// If this is the first request that's sent for this +/// particular root CID, then set `last_response` to `None`. +/// +/// On subsequent requests, set `last_response` to the +/// last successfully received response. +/// +/// Before actually sending the request over the network, +/// make sure to check the `request.indicates_finished()`. +/// If true, the client already has all data. +pub async fn request( + root: Cid, + last_response: Option, + config: &Config, + store: &impl BlockStore, +) -> Result { + Ok(block_receive(root, last_response, config, store) + .await? + .into()) +} + +/// Respond to a CAR mirror pull request. +pub async fn response( + root: Cid, + request: PullRequest, + config: &Config, + store: &impl BlockStore, +) -> Result { + let receiver_state = Some(ReceiverState::from(request)); + block_send(root, receiver_state, config, store).await +} + +#[cfg(test)] +mod tests { + use crate::{ + common::Config, + dag_walk::DagWalk, + test_utils::{setup_random_dag, Metrics}, + }; + use anyhow::Result; + use futures::TryStreamExt; + use libipld::Cid; + use std::collections::HashSet; + use wnfs_common::MemoryBlockStore; + + pub(crate) async fn simulate_protocol( + root: Cid, + config: &Config, + client_store: &MemoryBlockStore, + server_store: &MemoryBlockStore, + ) -> Result> { + let mut metrics = Vec::new(); + let mut request = crate::pull::request(root, None, config, client_store).await?; + loop { + let request_bytes = serde_ipld_dagcbor::to_vec(&request)?.len(); + let response = crate::pull::response(root, request, config, server_store).await?; + let response_bytes = response.bytes.len(); + + metrics.push(Metrics { + request_bytes, + response_bytes, + }); + + request = crate::pull::request(root, Some(response), config, client_store).await?; + if request.indicates_finished() { + break; + } + } + + Ok(metrics) + } + + #[async_std::test] + async fn test_transfer() -> Result<()> { + let client_store = &MemoryBlockStore::new(); + let (root, ref server_store) = setup_random_dag(256, 10 * 1024 /* 10 KiB */).await?; + + simulate_protocol(root, &Config::default(), client_store, server_store).await?; + + // client should have all data + let client_cids = DagWalk::breadth_first([root]) + .stream(client_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await?; + let server_cids = DagWalk::breadth_first([root]) + .stream(server_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await?; + + assert_eq!(client_cids, server_cids); + + Ok(()) + } +} + +#[cfg(test)] +mod proptests { + use crate::{ + common::Config, + dag_walk::DagWalk, + test_utils::{setup_blockstore, variable_blocksize_dag}, + }; + use futures::TryStreamExt; + use libipld::{Cid, Ipld}; + use std::collections::HashSet; + use test_strategy::proptest; + use wnfs_common::MemoryBlockStore; + + #[proptest] + fn cold_transfer_completes(#[strategy(variable_blocksize_dag())] dag: (Vec<(Cid, Ipld)>, Cid)) { + let (blocks, root) = dag; + async_std::task::block_on(async { + let server_store = &setup_blockstore(blocks).await.unwrap(); + let client_store = &MemoryBlockStore::new(); + + crate::pull::tests::simulate_protocol( + root, + &Config::default(), + client_store, + server_store, + ) + .await + .unwrap(); + + // client should have all data + let client_cids = DagWalk::breadth_first([root]) + .stream(client_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await + .unwrap(); + let server_cids = DagWalk::breadth_first([root]) + .stream(server_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await + .unwrap(); + + assert_eq!(client_cids, server_cids); + }) + } +} diff --git a/car-mirror/src/push.rs b/car-mirror/src/push.rs new file mode 100644 index 0000000..0fb229d --- /dev/null +++ b/car-mirror/src/push.rs @@ -0,0 +1,227 @@ +use crate::{ + common::{block_receive, block_send, CarFile, Config, ReceiverState}, + messages::PushResponse, +}; +use anyhow::Result; +use libipld_core::cid::Cid; +use wnfs_common::BlockStore; + +/// Create a CAR mirror push request. +/// +/// On the first request for a particular `root`, set +/// `last_response` to `None`. +/// +/// For subsequent requests, set it to the last successful +/// response from a request with the same `root`. +/// +/// The returned request body is a CAR file from some of the first +/// blocks below the root. +pub async fn request( + root: Cid, + last_response: Option, + config: &Config, + store: &impl BlockStore, +) -> Result { + let receiver_state = last_response.map(ReceiverState::from); + block_send(root, receiver_state, config, store).await +} + +/// Create a response for a CAR mirror push request. +/// +/// This takes in the CAR file from the request body and stores its blocks +/// in the given `store`, if the blocks can be shown to relate +/// to the `root` CID. +/// +/// Returns a response that gives the client information about what +/// other data remains to be fetched. +pub async fn response( + root: Cid, + request: CarFile, + config: &Config, + store: &impl BlockStore, +) -> Result { + Ok(block_receive(root, Some(request), config, store) + .await? + .into()) +} + +#[cfg(test)] +mod tests { + use crate::{ + common::Config, + dag_walk::DagWalk, + test_utils::{ + get_cid_at_approx_path, setup_random_dag, total_dag_blocks, total_dag_bytes, Metrics, + Rvg, + }, + }; + use anyhow::Result; + use futures::TryStreamExt; + use libipld::Cid; + use proptest::collection::vec; + use std::collections::HashSet; + use wnfs_common::MemoryBlockStore; + + pub(crate) async fn simulate_protocol( + root: Cid, + config: &Config, + client_store: &MemoryBlockStore, + server_store: &MemoryBlockStore, + ) -> Result> { + let mut metrics = Vec::new(); + let mut request = crate::push::request(root, None, config, client_store).await?; + loop { + let request_bytes = request.bytes.len(); + let response = crate::push::response(root, request, config, server_store).await?; + let response_bytes = serde_ipld_dagcbor::to_vec(&response)?.len(); + + metrics.push(Metrics { + request_bytes, + response_bytes, + }); + + if response.indicates_finished() { + break; + } + request = crate::push::request(root, Some(response), config, client_store).await?; + } + + Ok(metrics) + } + + #[async_std::test] + async fn test_transfer() -> Result<()> { + let (root, ref client_store) = setup_random_dag(256, 10 * 1024 /* 10 KiB */).await?; + let server_store = &MemoryBlockStore::new(); + simulate_protocol(root, &Config::default(), client_store, server_store).await?; + + // receiver should have all data + let client_cids = DagWalk::breadth_first([root]) + .stream(client_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await?; + let server_cids = DagWalk::breadth_first([root]) + .stream(server_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await?; + + assert_eq!(client_cids, server_cids); + + Ok(()) + } + + #[async_std::test] + async fn test_deduplicating_transfer() -> Result<()> { + let (root, ref client_store) = setup_random_dag(256, 10 * 1024 /* 10 KiB */).await?; + let total_bytes = total_dag_bytes(root, client_store).await?; + let path = Rvg::new().sample(&vec(0usize..128, 0..64)); + let second_root = get_cid_at_approx_path(path, root, client_store).await?; + + let server_store = &MemoryBlockStore::new(); + let config = &Config::default(); + let metrics1 = simulate_protocol(second_root, config, client_store, server_store).await?; + let metrics2 = simulate_protocol(root, config, client_store, server_store).await?; + + let total_network_bytes = metrics1 + .into_iter() + .chain(metrics2.into_iter()) + .map(|metric| metric.request_bytes + metric.response_bytes) + .sum::(); + + println!("Total DAG bytes: {total_bytes}"); + println!("Total network bytes: {total_network_bytes}"); + + Ok(()) + } + + #[async_std::test] + async fn print_metrics() -> Result<()> { + const TESTS: usize = 200; + const DAG_SIZE: u16 = 256; + const BLOCK_PADDING: usize = 10 * 1024; + + let mut total_rounds = 0; + let mut total_blocks = 0; + let mut total_block_bytes = 0; + let mut total_network_bytes = 0; + for _ in 0..TESTS { + let (root, ref client_store) = setup_random_dag(DAG_SIZE, BLOCK_PADDING).await?; + let server_store = &MemoryBlockStore::new(); + let metrics = + simulate_protocol(root, &Config::default(), client_store, server_store).await?; + + total_rounds += metrics.len(); + total_blocks += total_dag_blocks(root, client_store).await?; + total_block_bytes += total_dag_bytes(root, client_store).await?; + total_network_bytes += metrics + .iter() + .map(|metric| metric.request_bytes + metric.response_bytes) + .sum::(); + } + + println!( + "Average # of rounds: {}", + total_rounds as f64 / TESTS as f64 + ); + println!( + "Average # of blocks: {}", + total_blocks as f64 / TESTS as f64 + ); + println!( + "Average network overhead: {}%", + (total_network_bytes as f64 / total_block_bytes as f64 - 1.0) * 100.0 + ); + + Ok(()) + } +} + +#[cfg(test)] +mod proptests { + use crate::{ + common::Config, + dag_walk::DagWalk, + test_utils::{setup_blockstore, variable_blocksize_dag}, + }; + use futures::TryStreamExt; + use libipld::{Cid, Ipld}; + use std::collections::HashSet; + use test_strategy::proptest; + use wnfs_common::MemoryBlockStore; + + #[proptest] + fn cold_transfer_completes(#[strategy(variable_blocksize_dag())] dag: (Vec<(Cid, Ipld)>, Cid)) { + let (blocks, root) = dag; + async_std::task::block_on(async { + let client_store = &setup_blockstore(blocks).await.unwrap(); + let server_store = &MemoryBlockStore::new(); + + crate::push::tests::simulate_protocol( + root, + &Config::default(), + client_store, + server_store, + ) + .await + .unwrap(); + + // client should have all data + let client_cids = DagWalk::breadth_first([root]) + .stream(client_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await + .unwrap(); + let server_cids = DagWalk::breadth_first([root]) + .stream(server_store) + .map_ok(|(cid, _)| cid) + .try_collect::>() + .await + .unwrap(); + + assert_eq!(client_cids, server_cids); + }) + } +} diff --git a/car-mirror/src/test_utils/blockstore_utils.rs b/car-mirror/src/test_utils/blockstore_utils.rs new file mode 100644 index 0000000..3394271 --- /dev/null +++ b/car-mirror/src/test_utils/blockstore_utils.rs @@ -0,0 +1,64 @@ +use crate::common::references; +use anyhow::Result; +use bytes::Bytes; +use libipld::{Cid, Ipld, IpldCodec}; +use std::io::Write; +use wnfs_common::{dagcbor::encode, BlockStore, MemoryBlockStore}; + +/// Take a list of dag-cbor IPLD blocks and store all of them as dag-cbor in a +/// MemoryBlockStore & return it. +pub async fn setup_blockstore(blocks: Vec<(Cid, Ipld)>) -> Result { + let store = MemoryBlockStore::new(); + setup_existing_blockstore(blocks, &store).await?; + Ok(store) +} + +/// Take a list of dag-cbor IPLD blocks and store all of them as dag-cbor in +/// the given `BlockStore`. +pub async fn setup_existing_blockstore( + blocks: Vec<(Cid, Ipld)>, + store: &impl BlockStore, +) -> Result<()> { + for (cid, ipld) in blocks.into_iter() { + let block: Bytes = encode(&ipld)?.into(); + let cid_store = store.put_block(block, IpldCodec::DagCbor.into()).await?; + debug_assert_eq!(cid, cid_store); + } + + Ok(()) +} + +/// Print a DAG as a dot file with truncated CIDs +pub fn dag_to_dot( + writer: &mut impl Write, + blocks: impl IntoIterator, +) -> Result<()> { + writeln!(writer, "digraph {{")?; + + for (cid, ipld) in blocks { + let bytes = encode(&ipld)?; + let refs = references(cid, bytes, Vec::new())?; + for to_cid in refs { + print_truncated_string(writer, cid.to_string())?; + write!(writer, " -> ")?; + print_truncated_string(writer, to_cid.to_string())?; + writeln!(writer)?; + } + } + + writeln!(writer, "}}")?; + + Ok(()) +} + +fn print_truncated_string(writer: &mut impl Write, mut string: String) -> Result<()> { + if string.len() > 20 { + let mut string_rest = string.split_off(10); + let string_end = string_rest.split_off(std::cmp::max(string_rest.len(), 10) - 10); + write!(writer, "\"{string}...{string_end}\"")?; + } else { + write!(writer, "\"{string}\"")?; + } + + Ok(()) +} diff --git a/car-mirror/src/test_utils/dag_strategy.rs b/car-mirror/src/test_utils/dag_strategy.rs new file mode 100644 index 0000000..571d376 --- /dev/null +++ b/car-mirror/src/test_utils/dag_strategy.rs @@ -0,0 +1,110 @@ +use bytes::Bytes; +use libipld::{Cid, Ipld, IpldCodec}; +use libipld_core::multihash::{Code, MultihashDigest}; +use proptest::{prelude::Rng, strategy::Strategy, test_runner::TestRng}; +use roaring_graphs::{arb_dag, DirectedAcyclicGraph, Vertex}; +use std::{ + collections::{BTreeMap, HashSet}, + fmt::Debug, + ops::Range, +}; +use wnfs_common::dagcbor::encode; + +/// A strategy for use with proptest to generate random DAGs (directed acyclic graphs). +/// The strategy generates a list of blocks of type T and their CIDs, as well as +/// the root block's CID. +pub fn arb_ipld_dag( + vertex_count: impl Into>, + edge_probability: f64, + generate_block: impl Fn(Vec, &mut TestRng) -> (Cid, T) + Clone, +) -> impl Strategy, Cid)> { + arb_dag(vertex_count, edge_probability) + .prop_perturb(move |dag, mut rng| dag_to_nodes(&dag, &mut rng, generate_block.clone())) +} + +/// A block-generating function for use with `arb_ipld_dag`. +pub fn links_to_ipld(cids: Vec, _: &mut TestRng) -> (Cid, Ipld) { + let ipld = Ipld::List(cids.into_iter().map(Ipld::Link).collect()); + let bytes = encode(&ipld).unwrap(); + let cid = Cid::new_v1(IpldCodec::DagCbor.into(), Code::Blake3_256.digest(&bytes)); + (cid, ipld) +} + +/// A block-generating function for use with `arb_ipld_dag`. +pub fn links_to_dag_cbor(cids: Vec, _: &mut TestRng) -> (Cid, Bytes) { + let ipld = Ipld::List(cids.into_iter().map(Ipld::Link).collect()); + let bytes: Bytes = encode(&ipld).unwrap().into(); + let cid = Cid::new_v1(IpldCodec::DagCbor.into(), Code::Blake3_256.digest(&bytes)); + (cid, bytes) +} + +/// A block-generating function for use with `arb_ipld_dag`. +/// +/// Creates (a function that creates) an IPLD block with given links & some +/// random `padding_bytes` bytes attached. +pub fn links_to_padded_ipld( + padding_bytes: usize, +) -> impl Fn(Vec, &mut TestRng) -> (Cid, Ipld) + Clone { + move |cids, rng| { + let mut padding = Vec::with_capacity(padding_bytes); + for _ in 0..padding_bytes { + padding.push(rng.gen::()); + } + + let ipld = Ipld::Map(BTreeMap::from([ + ("data".into(), Ipld::Bytes(padding)), + ( + "links".into(), + Ipld::List(cids.into_iter().map(Ipld::Link).collect()), + ), + ])); + let bytes = encode(&ipld).unwrap(); + let cid = Cid::new_v1(IpldCodec::DagCbor.into(), Code::Blake3_256.digest(&bytes)); + (cid, ipld) + } +} + +/// Turn a directed acyclic graph into a list of nodes (with their CID) and a root CID. +/// This will select only the DAG that's reachable from the root. +pub fn dag_to_nodes( + dag: &DirectedAcyclicGraph, + rng: &mut TestRng, + generate_node: impl Fn(Vec, &mut TestRng) -> (Cid, T) + Clone, +) -> (Vec<(Cid, T)>, Cid) { + let mut blocks = Vec::new(); + let mut visited = HashSet::new(); + let (cid, block) = dag_to_nodes_helper(dag, 0, rng, generate_node, &mut blocks, &mut visited); + blocks.push((cid, block)); + (blocks, cid) +} + +fn dag_to_nodes_helper( + dag: &DirectedAcyclicGraph, + root: Vertex, + rng: &mut TestRng, + generate_node: impl Fn(Vec, &mut TestRng) -> (Cid, T) + Clone, + arr: &mut Vec<(Cid, T)>, + visited: &mut HashSet, +) -> (Cid, T) { + let mut child_blocks = Vec::new(); + if root >= dag.get_vertex_count() { + println!("{root}, {}", dag.get_vertex_count()); + } + for child in dag.iter_children(root) { + if visited.contains(&child) { + continue; + } + visited.insert(child); + child_blocks.push(dag_to_nodes_helper( + dag, + child, + rng, + generate_node.clone(), + arr, + visited, + )); + } + let result = generate_node(child_blocks.iter().map(|(cid, _)| *cid).collect(), rng); + arr.extend(child_blocks); + result +} diff --git a/car-mirror/src/test_utils/local_utils.rs b/car-mirror/src/test_utils/local_utils.rs new file mode 100644 index 0000000..51f021d --- /dev/null +++ b/car-mirror/src/test_utils/local_utils.rs @@ -0,0 +1,88 @@ +///! Crate-local test utilities +use super::{arb_ipld_dag, links_to_padded_ipld, setup_blockstore, Rvg}; +use crate::{common::references, dag_walk::DagWalk}; +use anyhow::Result; +use futures::TryStreamExt; +use libipld::{Cid, Ipld}; +use proptest::strategy::Strategy; +use wnfs_common::{BlockStore, MemoryBlockStore}; + +#[derive(Clone, Debug)] +pub(crate) struct Metrics { + pub(crate) request_bytes: usize, + pub(crate) response_bytes: usize, +} + +/// Walk a root DAG along some path. +/// At each node, take the `n % numlinks`th link, +/// and only walk the path as long as there are further links. +pub(crate) async fn get_cid_at_approx_path( + path: Vec, + root: Cid, + store: &impl BlockStore, +) -> Result { + let mut working_cid = root; + for nth in path { + let block = store.get_block(&working_cid).await?; + let refs = references(working_cid, block, Vec::new())?; + if refs.is_empty() { + break; + } + + working_cid = refs[nth % refs.len()]; + } + Ok(working_cid) +} + +pub(crate) fn padded_dag_strategy( + dag_size: u16, + block_padding: usize, +) -> impl Strategy, Cid)> { + arb_ipld_dag(1..dag_size, 0.5, links_to_padded_ipld(block_padding)) +} + +pub(crate) fn variable_blocksize_dag() -> impl Strategy, Cid)> { + const MAX_DAG_NODES: u16 = 128; // with this proptests run ~15 sec for me + const MAX_LINK_BYTES: usize = MAX_DAG_NODES as usize * 42; // 1 byte cbor CID tag, 1 byte multibase indicator, 40 bytes CID + + // 1 byte cbor tag for whole object, + // 1 byte cbor tag for block padding bytes + // up to ~3 bytes for block padding size + // 1 bytes cbor tag for list (of cids) + // up to ~2 bytes for list size + const EST_OVERHEAD: usize = 1 + 1 + 3 + 1 + 2; + const MAX_BLOCK_SIZE: usize = 256 * 1024; + const MAX_BLOCK_PADDING: usize = MAX_BLOCK_SIZE - EST_OVERHEAD - MAX_LINK_BYTES; + + (32..MAX_BLOCK_PADDING).prop_ind_flat_map(move |block_padding| { + arb_ipld_dag(1..MAX_DAG_NODES, 0.5, links_to_padded_ipld(block_padding)) + }) +} + +pub(crate) async fn setup_random_dag( + dag_size: u16, + block_padding: usize, +) -> Result<(Cid, MemoryBlockStore)> { + let (blocks, root) = Rvg::new().sample(&padded_dag_strategy(dag_size, block_padding)); + let store = setup_blockstore(blocks).await?; + Ok((root, store)) +} + +pub(crate) async fn total_dag_bytes(root: Cid, store: &impl BlockStore) -> Result { + Ok(DagWalk::breadth_first([root]) + .stream(store) + .map_ok(|(_, block)| block.len()) + .try_collect::>() + .await? + .into_iter() + .sum::()) +} + +pub(crate) async fn total_dag_blocks(root: Cid, store: &impl BlockStore) -> Result { + Ok(DagWalk::breadth_first([root]) + .stream(store) + .map_ok(|(_, block)| block.len()) + .try_collect::>() + .await? + .len()) +} diff --git a/car-mirror/src/test_utils/mod.rs b/car-mirror/src/test_utils/mod.rs index 4a30e2a..11cc566 100644 --- a/car-mirror/src/test_utils/mod.rs +++ b/car-mirror/src/test_utils/mod.rs @@ -1,5 +1,18 @@ +#[cfg(feature = "test_utils")] +mod dag_strategy; /// Random value generator for sampling data. #[cfg(feature = "test_utils")] mod rvg; #[cfg(feature = "test_utils")] +pub use dag_strategy::*; +#[cfg(feature = "test_utils")] pub use rvg::*; +#[cfg(feature = "test_utils")] +mod blockstore_utils; +#[cfg(feature = "test_utils")] +pub use blockstore_utils::*; + +#[cfg(test)] +mod local_utils; +#[cfg(test)] +pub(crate) use local_utils::*; diff --git a/car-mirror/tests/integration_test.rs b/car-mirror/tests/integration_test.rs deleted file mode 100644 index d60e3f3..0000000 --- a/car-mirror/tests/integration_test.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[test] -fn test_add() { - assert_eq!(car_mirror::add(3, 2), 5); -} diff --git a/deny.toml b/deny.toml index f532111..a6c5ca1 100644 --- a/deny.toml +++ b/deny.toml @@ -76,7 +76,8 @@ allow = [ "BSD-2-Clause", "BSD-3-Clause", "ISC", - "Zlib" + "Zlib", + "BSL-1.0" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses