diff --git a/Cargo.lock b/Cargo.lock index f08f82e85..b7dc9aab4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,9 +64,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -113,9 +113,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "append-only-vec" @@ -159,7 +159,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -170,7 +170,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -356,7 +356,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.86", + "syn 2.0.87", "which", ] @@ -535,9 +535,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" dependencies = [ "jobserver", "libc", @@ -679,7 +679,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f76990911f2267d837d9d0ad060aa63aaad170af40904b29461734c339030d4d" dependencies = [ "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -706,9 +706,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -880,6 +880,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "document-features" version = "0.2.10" @@ -925,7 +936,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -973,9 +984,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fd-lock" @@ -1123,7 +1134,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1186,7 +1197,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1280,9 +1291,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -1414,9 +1425,9 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ "hyper", "hyper-util", @@ -1483,14 +1494,143 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1522,7 +1662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -1658,9 +1798,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "libloading" @@ -1701,6 +1841,7 @@ dependencies = [ "log", "orchard", "proptest", + "rustls", "sapling-crypto", "serde_json", "shardtree", @@ -1722,6 +1863,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "litrs" version = "0.4.1" @@ -2009,7 +2156,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2189,7 +2336,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2252,7 +2399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2274,7 +2421,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2333,7 +2480,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.86", + "syn 2.0.87", "tempfile", ] @@ -2347,7 +2494,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2551,9 +2698,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -2662,7 +2809,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.86", + "syn 2.0.87", "walkdir", ] @@ -2690,9 +2837,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ "bitflags 2.6.0", "errno", @@ -2911,9 +3058,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -2921,9 +3068,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] @@ -2940,13 +3087,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3087,6 +3234,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3124,9 +3277,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3148,6 +3301,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -3187,9 +3351,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -3216,7 +3380,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3227,28 +3391,28 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "test-case-core", ] [[package]] name = "thiserror" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3290,6 +3454,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -3307,9 +3481,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes 1.8.0", @@ -3331,7 +3505,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3424,7 +3598,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3493,7 +3667,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3570,12 +3744,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" @@ -3636,15 +3804,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -3677,7 +3857,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3736,7 +3916,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -3770,7 +3950,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4036,6 +4216,18 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -4051,6 +4243,30 @@ version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zcash_address" version = "0.6.0" @@ -4255,7 +4471,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -4275,7 +4512,29 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] diff --git a/libtonode-tests/Cargo.toml b/libtonode-tests/Cargo.toml index cf77142d3..d8e62887b 100644 --- a/libtonode-tests/Cargo.toml +++ b/libtonode-tests/Cargo.toml @@ -33,3 +33,6 @@ tempfile.workspace = true tracing-subscriber.workspace = true proptest.workspace = true bech32 = "0.11.0" + +[dev-dependencies] +rustls.workspace = true diff --git a/libtonode-tests/tests/sync.rs b/libtonode-tests/tests/sync.rs index c7197614c..6f861688c 100644 --- a/libtonode-tests/tests/sync.rs +++ b/libtonode-tests/tests/sync.rs @@ -12,6 +12,9 @@ use zingolib::{ #[ignore = "too slow, and flakey"] #[tokio::test] async fn sync_mainnet_test() { + rustls::crypto::ring::default_provider() + .install_default() + .expect("Ring to work as a default"); tracing_subscriber::fmt().init(); let uri = construct_lightwalletd_uri(Some(DEFAULT_LIGHTWALLETD_SERVER.to_string())); @@ -62,6 +65,7 @@ async fn sync_test() { .await .unwrap(); + dbg!(recipient.wallet.wallet_transactions()); dbg!(recipient.wallet.wallet_blocks()); dbg!(recipient.wallet.nullifier_map()); dbg!(recipient.wallet.sync_state()); diff --git a/zingo-sync/src/client.rs b/zingo-sync/src/client.rs index beaea957e..fa556a493 100644 --- a/zingo-sync/src/client.rs +++ b/zingo-sync/src/client.rs @@ -2,6 +2,8 @@ use std::ops::Range; +use tokio::sync::{mpsc::UnboundedSender, oneshot}; + use zcash_client_backend::{ data_api::chain::ChainState, proto::{ @@ -9,9 +11,10 @@ use zcash_client_backend::{ service::{BlockId, TreeState}, }, }; -use zcash_primitives::consensus::BlockHeight; - -use tokio::sync::{mpsc::UnboundedSender, oneshot}; +use zcash_primitives::{ + consensus::BlockHeight, + transaction::{Transaction, TxId}, +}; pub mod fetch; @@ -26,6 +29,8 @@ pub enum FetchRequest { CompactBlockRange(oneshot::Sender>, Range), /// Gets the tree states for a specified block height.. TreeState(oneshot::Sender, BlockHeight), + /// Get a full transaction by txid. + Transaction(oneshot::Sender<(Transaction, BlockHeight)>, TxId), } /// Gets the height of the blockchain from the server. @@ -34,7 +39,7 @@ pub enum FetchRequest { pub async fn get_chain_height( fetch_request_sender: UnboundedSender, ) -> Result { - let (sender, receiver) = oneshot::channel::(); + let (sender, receiver) = oneshot::channel(); fetch_request_sender .send(FetchRequest::ChainTip(sender)) .unwrap(); @@ -49,7 +54,7 @@ pub async fn get_compact_block_range( fetch_request_sender: UnboundedSender, block_range: Range, ) -> Result, ()> { - let (sender, receiver) = oneshot::channel::>(); + let (sender, receiver) = oneshot::channel(); fetch_request_sender .send(FetchRequest::CompactBlockRange(sender, block_range)) .unwrap(); @@ -57,14 +62,14 @@ pub async fn get_compact_block_range( Ok(compact_blocks) } -/// Gets the frontiers for a specified block height.. +/// Gets the frontiers for a specified block height. /// /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. pub async fn get_frontiers( fetch_request_sender: UnboundedSender, block_height: BlockHeight, ) -> Result { - let (sender, receiver) = oneshot::channel::(); + let (sender, receiver) = oneshot::channel(); fetch_request_sender .send(FetchRequest::TreeState(sender, block_height)) .unwrap(); @@ -73,3 +78,18 @@ pub async fn get_frontiers( Ok(frontiers) } +/// Gets a full transaction for a specified txid. +/// +/// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. +pub async fn get_transaction_and_block_height( + fetch_request_sender: UnboundedSender, + txid: TxId, +) -> Result<(Transaction, BlockHeight), ()> { + let (sender, receiver) = oneshot::channel(); + fetch_request_sender + .send(FetchRequest::Transaction(sender, txid)) + .unwrap(); + let transaction_and_block_height = receiver.await.unwrap(); + + Ok(transaction_and_block_height) +} diff --git a/zingo-sync/src/client/fetch.rs b/zingo-sync/src/client/fetch.rs index c0f4be60b..1510b918f 100644 --- a/zingo-sync/src/client/fetch.rs +++ b/zingo-sync/src/client/fetch.rs @@ -8,10 +8,13 @@ use zcash_client_backend::proto::{ compact_formats::CompactBlock, service::{ compact_tx_streamer_client::CompactTxStreamerClient, BlockId, BlockRange, ChainSpec, - TreeState, + TreeState, TxFilter, }, }; -use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::{ + consensus::{self, BlockHeight, BranchId}, + transaction::{Transaction, TxId}, +}; use crate::client::FetchRequest; @@ -24,6 +27,7 @@ use crate::client::FetchRequest; pub async fn fetch( mut fetch_request_receiver: UnboundedReceiver, mut client: CompactTxStreamerClient, + consensus_parameters: impl consensus::Parameters, ) -> Result<(), ()> { let mut fetch_request_queue: Vec = Vec::new(); @@ -37,7 +41,9 @@ pub async fn fetch( let fetch_request = select_fetch_request(&mut fetch_request_queue); if let Some(request) = fetch_request { - fetch_from_server(&mut client, request).await.unwrap(); + fetch_from_server(&mut client, &consensus_parameters, request) + .await + .unwrap(); } } } @@ -91,24 +97,30 @@ fn select_fetch_request(fetch_request_queue: &mut Vec) -> Option, + parameters: &impl consensus::Parameters, fetch_request: FetchRequest, ) -> Result<(), ()> { match fetch_request { FetchRequest::ChainTip(sender) => { tracing::info!("Fetching chain tip."); - let block_id = get_latest_block(client).await; + let block_id = get_latest_block(client).await.unwrap(); sender.send(block_id).unwrap(); } FetchRequest::CompactBlockRange(sender, block_range) => { tracing::info!("Fetching compact blocks. {:?}", &block_range); - let compact_blocks = get_block_range(client, block_range).await; + let compact_blocks = get_block_range(client, block_range).await.unwrap(); sender.send(compact_blocks).unwrap(); } FetchRequest::TreeState(sender, block_height) => { tracing::info!("Fetching tree state. {:?}", &block_height); - let tree_state = get_tree_state(client, block_height).await; + let tree_state = get_tree_state(client, block_height).await.unwrap(); sender.send(tree_state).unwrap(); } + FetchRequest::Transaction(sender, txid) => { + tracing::info!("Fetching transaction. {:?}", txid); + let transaction = get_transaction(client, parameters, txid).await.unwrap(); + sender.send(transaction).unwrap(); + } } Ok(()) @@ -116,15 +128,15 @@ async fn fetch_from_server( async fn get_latest_block( client: &mut CompactTxStreamerClient, -) -> BlockId { +) -> Result { let request = tonic::Request::new(ChainSpec {}); - client.get_latest_block(request).await.unwrap().into_inner() + Ok(client.get_latest_block(request).await.unwrap().into_inner()) } async fn get_block_range( client: &mut CompactTxStreamerClient, block_range: Range, -) -> Vec { +) -> Result, ()> { let mut compact_blocks: Vec = Vec::with_capacity(u64::from(block_range.end - block_range.start) as usize); @@ -144,16 +156,39 @@ async fn get_block_range( compact_blocks.push(compact_block); } - compact_blocks + Ok(compact_blocks) } async fn get_tree_state( client: &mut CompactTxStreamerClient, block_height: BlockHeight, -) -> TreeState { +) -> Result { let request = tonic::Request::new(BlockId { height: block_height.into(), hash: vec![], }); - client.get_tree_state(request).await.unwrap().into_inner() + Ok(client.get_tree_state(request).await.unwrap().into_inner()) +} + +async fn get_transaction( + client: &mut CompactTxStreamerClient, + parameters: &impl consensus::Parameters, + txid: TxId, +) -> Result<(Transaction, BlockHeight), ()> { + let request = tonic::Request::new(TxFilter { + block: None, + index: 0, + hash: txid.as_ref().to_vec(), + }); + + let raw_transaction = client.get_transaction(request).await.unwrap().into_inner(); + let block_height = BlockHeight::from_u32(raw_transaction.height as u32); + + let transaction = Transaction::read( + &raw_transaction.data[..], + BranchId::for_height(parameters, block_height), + ) + .unwrap(); + + Ok((transaction, block_height)) } diff --git a/zingo-sync/src/lib.rs b/zingo-sync/src/lib.rs index da3250d43..24905594b 100644 --- a/zingo-sync/src/lib.rs +++ b/zingo-sync/src/lib.rs @@ -4,10 +4,10 @@ //! Entrypoint: [`crate::sync::sync`] pub mod client; -pub mod interface; pub(crate) mod keys; #[allow(missing_docs)] pub mod primitives; pub(crate) mod scan; pub mod sync; +pub mod traits; pub mod witness; diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs index b609a1535..de5f45ff9 100644 --- a/zingo-sync/src/primitives.rs +++ b/zingo-sync/src/primitives.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use getset::{CopyGetters, Getters, MutGetters}; +use incrementalmerkletree::Position; use zcash_client_backend::data_api::scanning::ScanRange; use zcash_primitives::{block::BlockHash, consensus::BlockHeight, transaction::TxId}; @@ -30,7 +31,7 @@ impl Default for SyncState { } /// Output ID for a given pool type -#[derive(PartialEq, Eq, Hash, Clone, Copy, CopyGetters)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, CopyGetters)] #[getset(get_copy = "pub")] pub struct OutputId { /// ID of associated transaction @@ -108,3 +109,132 @@ impl WalletBlock { &self.txids } } + +/// Wallet transaction +#[derive(Debug, CopyGetters)] +#[getset(get_copy = "pub")] +pub struct WalletTransaction { + #[getset(get_copy = "pub")] + txid: TxId, + #[getset(get_copy = "pub")] + block_height: BlockHeight, + #[getset(skip)] + sapling_notes: Vec, + #[getset(skip)] + orchard_notes: Vec, +} + +impl WalletTransaction { + pub fn from_parts( + txid: TxId, + block_height: BlockHeight, + sapling_notes: Vec, + orchard_notes: Vec, + ) -> Self { + Self { + txid, + block_height, + sapling_notes, + orchard_notes, + } + } + + pub fn sapling_notes(&self) -> &[SaplingNote] { + &self.sapling_notes + } + + pub fn orchard_notes(&self) -> &[OrchardNote] { + &self.orchard_notes + } +} + +// TODO: change memo to correct type + +#[derive(Debug, Getters, CopyGetters)] +pub struct SaplingNote { + #[getset(get_copy = "pub")] + output_id: OutputId, + #[getset(get = "pub")] + note: sapling_crypto::Note, + #[getset(get_copy = "pub")] + nullifier: sapling_crypto::Nullifier, //TODO: make option and add handling for syncing without nullfiier deriving key + #[getset(get_copy = "pub")] + position: Position, + #[getset(get_copy = "pub")] + memo: [u8; 512], +} + +impl SyncNote for SaplingNote { + type WalletNote = Self; + type ZcashNote = sapling_crypto::Note; + type Nullifier = sapling_crypto::Nullifier; + type Memo = [u8; 512]; + + fn from_parts( + output_id: OutputId, + note: Self::ZcashNote, + nullifier: Self::Nullifier, + position: Position, + memo: Self::Memo, + ) -> Self::WalletNote { + Self { + output_id, + note, + nullifier, + position, + memo, + } + } +} + +#[derive(Debug, Getters, CopyGetters)] +pub struct OrchardNote { + #[getset(get_copy = "pub")] + output_id: OutputId, + #[getset(get = "pub")] + note: orchard::Note, + #[getset(get_copy = "pub")] + nullifier: orchard::note::Nullifier, //TODO: make option and add handling for syncing without nullfiier deriving key + #[getset(get_copy = "pub")] + position: Position, + #[getset(get_copy = "pub")] + memo: [u8; 512], +} + +impl SyncNote for OrchardNote { + type WalletNote = Self; + type ZcashNote = orchard::Note; + type Nullifier = orchard::note::Nullifier; + type Memo = [u8; 512]; + + fn from_parts( + output_id: OutputId, + note: Self::ZcashNote, + nullifier: Self::Nullifier, + position: Position, + memo: Self::Memo, + ) -> Self::WalletNote { + Self { + output_id, + note, + nullifier, + position, + memo, + } + } +} + +pub trait SyncNote { + type WalletNote; + type ZcashNote; + type Nullifier: Copy; + type Memo; + + fn from_parts( + output_id: OutputId, + note: Self::ZcashNote, + nullifier: Self::Nullifier, + position: Position, + memo: Self::Memo, + ) -> Self::WalletNote; +} diff --git a/zingo-sync/src/scan.rs b/zingo-sync/src/scan.rs index 820bc8393..0ea57dfe7 100644 --- a/zingo-sync/src/scan.rs +++ b/zingo-sync/src/scan.rs @@ -4,24 +4,35 @@ use std::{ }; use incrementalmerkletree::{Marking, Position, Retention}; -use orchard::{note_encryption::CompactAction, tree::MerkleHashOrchard}; -use sapling_crypto::{note_encryption::CompactOutputDescription, Node}; +use orchard::{ + note_encryption::{CompactAction, OrchardDomain}, + primitives::redpallas::{Signature, SpendAuth}, + tree::MerkleHashOrchard, + Action, +}; +use sapling_crypto::{ + bundle::{GrothProofBytes, OutputDescription}, + note_encryption::{CompactOutputDescription, SaplingDomain}, + Node, +}; use tokio::sync::mpsc; use zcash_client_backend::{ data_api::scanning::ScanRange, proto::compact_formats::{CompactBlock, CompactOrchardAction, CompactSaplingOutput, CompactTx}, }; -use zcash_note_encryption::Domain; +use zcash_note_encryption::{batch, BatchDomain, Domain, ShieldedOutput, ENC_CIPHERTEXT_SIZE}; use zcash_primitives::{ block::BlockHash, consensus::{BlockHeight, NetworkUpgrade, Parameters}, - transaction::TxId, + transaction::{components::sapling::zip212_enforcement, Transaction, TxId}, }; use crate::{ client::{self, get_compact_block_range, FetchRequest}, keys::{KeyId, ScanningKeyOps, ScanningKeys}, - primitives::{NullifierMap, OutputId, WalletBlock}, + primitives::{ + NullifierMap, OrchardNote, OutputId, SaplingNote, SyncNote, WalletBlock, WalletTransaction, + }, witness::ShardTreeData, }; @@ -115,7 +126,7 @@ impl InitialScanData { struct ScanData { pub(crate) nullifiers: NullifierMap, pub(crate) wallet_blocks: BTreeMap, - pub(crate) relevent_txids: HashSet, + pub(crate) relevant_txids: HashSet, pub(crate) decrypted_note_data: DecryptedNoteData, pub(crate) shard_tree_data: ShardTreeData, } @@ -123,6 +134,7 @@ struct ScanData { pub(crate) struct ScanResults { pub(crate) nullifiers: NullifierMap, pub(crate) wallet_blocks: BTreeMap, + pub(crate) wallet_transactions: HashMap, pub(crate) shard_tree_data: ShardTreeData, } @@ -146,8 +158,8 @@ impl Default for DecryptedNoteData { } } -// scans a given range and returns all data relevent to the specified keys -// `previous_wallet_block` is the block with height [scan_range.start - 1] +// scans a given range and returns all data relevant to the specified keys +// `previous_wallet_block` is the wallet block with height [scan_range.start - 1] pub(crate) async fn scan

( fetch_request_sender: mpsc::UnboundedSender, parameters: &P, @@ -166,7 +178,7 @@ where .unwrap(); let initial_scan_data = InitialScanData::new( - fetch_request_sender, + fetch_request_sender.clone(), parameters, compact_blocks .first() @@ -182,21 +194,30 @@ where let ScanData { nullifiers, wallet_blocks, - relevent_txids: _, - decrypted_note_data: _, + relevant_txids, + decrypted_note_data, shard_tree_data, } = scan_data; - // TODO: scan transactions + let wallet_transactions = scan_transactions( + fetch_request_sender, + parameters, + scanning_keys, + relevant_txids, + decrypted_note_data, + &wallet_blocks, + ) + .await + .unwrap(); Ok(ScanResults { nullifiers, wallet_blocks, + wallet_transactions, shard_tree_data, }) } -#[allow(clippy::type_complexity)] fn scan_compact_blocks

( compact_blocks: Vec, parameters: &P, @@ -212,7 +233,7 @@ where let mut wallet_blocks: BTreeMap = BTreeMap::new(); let mut nullifiers = NullifierMap::new(); - let mut relevent_txids: HashSet = HashSet::new(); + let mut relevant_txids: HashSet = HashSet::new(); let mut decrypted_note_data = DecryptedNoteData::new(); let mut shard_tree_data = ShardTreeData::new( Position::from(u64::from(initial_scan_data.sapling_initial_tree_size)), @@ -231,16 +252,16 @@ where .orchard .collect_results(block.hash(), transaction.txid()); - // gather the txids of all transactions relevent to the wallet + // gather the txids of all transactions relevant to the wallet // the edge case of transactions that this capability created but did not receive change // or create outgoing data is handled when the nullifiers are added and linked incoming_sapling_outputs.iter().for_each(|(output_id, _)| { - relevent_txids.insert(output_id.txid()); + relevant_txids.insert(output_id.txid()); }); incoming_orchard_outputs.iter().for_each(|(output_id, _)| { - relevent_txids.insert(output_id.txid()); + relevant_txids.insert(output_id.txid()); }); - // TODO: add outgoing outputs to relevent txids + // TODO: add outgoing outputs to relevant txids collect_nullifiers(&mut nullifiers, block.height(), transaction).unwrap(); @@ -301,7 +322,7 @@ where Ok(ScanData { nullifiers, wallet_blocks, - relevent_txids, + relevant_txids, decrypted_note_data, shard_tree_data, }) @@ -532,3 +553,143 @@ fn collect_nullifiers( }); Ok(()) } + +async fn scan_transactions

( + fetch_request_sender: mpsc::UnboundedSender, + parameters: &P, + scanning_keys: &ScanningKeys, + relevant_txids: HashSet, + decrypted_note_data: DecryptedNoteData, + wallet_blocks: &BTreeMap, +) -> Result, ()> +where + P: Parameters, +{ + let mut wallet_transactions = HashMap::with_capacity(relevant_txids.len()); + + for txid in relevant_txids { + let (transaction, block_height) = + client::get_transaction_and_block_height(fetch_request_sender.clone(), txid) + .await + .unwrap(); + + if transaction.txid() != txid { + panic!("transaction txid does not match txid requested!") + } + + // wallet block must exist, otherwise the transaction will not have access to essential data such as the time it was mined + if let Some(wallet_block) = wallet_blocks.get(&block_height) { + if !wallet_block.txids().contains(&transaction.txid()) { + panic!("txid is not found in the wallet block at the transaction height!"); + } + } else { + panic!("wallet block at transaction height not found!"); + } + + let wallet_transaction = scan_transaction( + parameters, + scanning_keys, + transaction, + block_height, + &decrypted_note_data, + ) + .unwrap(); + wallet_transactions.insert(txid, wallet_transaction); + } + + Ok(wallet_transactions) +} + +fn scan_transaction

( + parameters: &P, + scanning_keys: &ScanningKeys, + transaction: Transaction, + block_height: BlockHeight, + decrypted_note_data: &DecryptedNoteData, +) -> Result +where + P: Parameters, +{ + // TODO: price? + let zip212_enforcement = zip212_enforcement(parameters, block_height); + let mut sapling_notes: Vec = Vec::new(); + let mut orchard_notes: Vec = Vec::new(); + + if let Some(bundle) = transaction.sapling_bundle() { + let sapling_keys: Vec = scanning_keys + .sapling() + .iter() + .map(|(_, key)| key.prepare()) + .collect(); + let sapling_outputs: Vec<(SaplingDomain, OutputDescription)> = bundle + .shielded_outputs() + .iter() + .map(|output| (SaplingDomain::new(zip212_enforcement), output.clone())) + .collect(); + + scan_notes::, SaplingNote>( + &mut sapling_notes, + transaction.txid(), + &sapling_keys, + &sapling_outputs, + &decrypted_note_data.sapling_nullifiers_and_positions, + ) + .unwrap(); + } + + if let Some(bundle) = transaction.orchard_bundle() { + let orchard_keys: Vec = scanning_keys + .orchard() + .iter() + .map(|(_, key)| key.prepare()) + .collect(); + let orchard_actions: Vec<(OrchardDomain, Action>)> = bundle + .actions() + .iter() + .map(|action| (OrchardDomain::for_action(action), action.clone())) + .collect(); + + scan_notes::>, OrchardNote>( + &mut orchard_notes, + transaction.txid(), + &orchard_keys, + &orchard_actions, + &decrypted_note_data.orchard_nullifiers_and_positions, + ) + .unwrap(); + } + + Ok(WalletTransaction::from_parts( + transaction.txid(), + block_height, + sapling_notes, + orchard_notes, + )) +} + +fn scan_notes( + wallet_notes: &mut Vec, + txid: TxId, + ivks: &[D::IncomingViewingKey], + outputs: &[(D, Op)], + nullifiers_and_positions: &HashMap, +) -> Result<(), ()> +where + D: BatchDomain, + Op: ShieldedOutput, + N: SyncNote, +{ + for (output_index, output) in batch::try_note_decryption(ivks, outputs) + .into_iter() + .enumerate() + { + if let Some(((note, _, memo), _)) = output { + let output_id = OutputId::from_parts(txid, output_index); + let (nullifier, position) = nullifiers_and_positions.get(&output_id).unwrap(); + let wallet_note = N::from_parts(output_id, note, *nullifier, *position, memo); + wallet_notes.push(wallet_note); + } + } + + Ok(()) +} diff --git a/zingo-sync/src/scan/workers.rs b/zingo-sync/src/scan/workers.rs index d8a785d4e..4b0fa0998 100644 --- a/zingo-sync/src/scan/workers.rs +++ b/zingo-sync/src/scan/workers.rs @@ -74,7 +74,7 @@ where pub(crate) fn add_scan_task(&self, scan_task: ScanTask) -> Result<(), ()> { if let Some(worker) = self.workers.iter().find(|worker| !worker.is_scanning()) { - worker.add_scan_task(scan_task); + worker.add_scan_task(scan_task).unwrap(); } else { panic!("no idle workers!") } @@ -94,8 +94,10 @@ impl WorkerHandle { self.is_scanning.load(atomic::Ordering::Acquire) } - fn add_scan_task(&self, scan_task: ScanTask) { + fn add_scan_task(&self, scan_task: ScanTask) -> Result<(), ()> { self.scan_task_sender.send(scan_task).unwrap(); + + Ok(()) } } diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index 5558041a5..f27ef64f2 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -6,19 +6,18 @@ use std::time::Duration; use crate::client::FetchRequest; use crate::client::{fetch::fetch, get_chain_height}; -use crate::interface::{SyncBlocks, SyncNullifiers, SyncShardTrees, SyncWallet}; use crate::primitives::SyncState; use crate::scan::workers::{ScanTask, Scanner}; use crate::scan::ScanResults; +use crate::traits::{SyncBlocks, SyncNullifiers, SyncShardTrees, SyncTransactions, SyncWallet}; use tokio::sync::mpsc::error::TryRecvError; use zcash_client_backend::{ data_api::scanning::{ScanPriority, ScanRange}, proto::service::compact_tx_streamer_client::CompactTxStreamerClient, }; -use zcash_primitives::consensus::{BlockHeight, NetworkUpgrade, Parameters}; +use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; -use futures::future::try_join_all; use tokio::sync::mpsc; const BATCH_SIZE: u32 = 10; @@ -27,25 +26,26 @@ const BATCH_SIZE: u32 = 10; /// Syncs a wallet to the latest state of the blockchain pub async fn sync( client: CompactTxStreamerClient, - parameters: &P, + consensus_parameters: &P, wallet: &mut W, ) -> Result<(), ()> where - P: Parameters + Sync + Send + 'static, - W: SyncWallet + SyncBlocks + SyncNullifiers + SyncShardTrees, + P: consensus::Parameters + Sync + Send + 'static, + W: SyncWallet + SyncBlocks + SyncTransactions + SyncNullifiers + SyncShardTrees, { tracing::info!("Syncing wallet..."); - let mut handles = Vec::new(); - // create channel for sending fetch requests and launch fetcher task let (fetch_request_sender, fetch_request_receiver) = mpsc::unbounded_channel(); - let fetcher_handle = tokio::spawn(fetch(fetch_request_receiver, client)); - handles.push(fetcher_handle); + let fetcher_handle = tokio::spawn(fetch( + fetch_request_receiver, + client, + consensus_parameters.clone(), + )); update_scan_ranges( fetch_request_sender.clone(), - parameters, + consensus_parameters, wallet.get_birthday().unwrap(), wallet.get_sync_state_mut().unwrap(), ) @@ -58,7 +58,7 @@ where let mut scanner = Scanner::new( scan_results_sender, fetch_request_sender, - parameters.clone(), + consensus_parameters.clone(), ufvks, ); scanner.spawn_workers(); @@ -71,6 +71,8 @@ where if scanner.is_worker_idle() { if let Some(scan_range) = prepare_next_scan_range(wallet.get_sync_state_mut().unwrap()) { + // TODO: Can previous_wallet_block have the same value + // for more than one call to get_wallet_block? let previous_wallet_block = wallet .get_wallet_block(scan_range.block_range().start - 1) .ok(); @@ -104,7 +106,7 @@ where mark_scanned(scan_range, wallet.get_sync_state_mut().unwrap()).unwrap(); } - try_join_all(handles).await.unwrap(); + let _ = fetcher_handle.await.unwrap(); Ok(()) } @@ -117,7 +119,7 @@ async fn update_scan_ranges

( sync_state: &mut SyncState, ) -> Result<(), ()> where - P: Parameters, + P: consensus::Parameters, { let chain_height = get_chain_height(fetch_request_sender).await.unwrap(); @@ -141,6 +143,7 @@ where }; if wallet_height > chain_height { + // TODO: Isn't this a possible state if there's been a reorg? panic!("wallet is ahead of server!") } @@ -213,18 +216,21 @@ fn mark_scanned(scan_range: ScanRange, sync_state: &mut SyncState) -> Result<(), fn update_wallet_data(wallet: &mut W, scan_results: ScanResults) -> Result<(), ()> where - W: SyncBlocks + SyncNullifiers + SyncShardTrees, + W: SyncBlocks + SyncTransactions + SyncNullifiers + SyncShardTrees, { let ScanResults { nullifiers, wallet_blocks, + wallet_transactions, shard_tree_data, } = scan_results; - // TODO: if scan priority is historic, retain only relevent blocks and nullifiers as we have all information and requires a lot of memory / storage - // must still retain top 100 blocks for re-org purposes wallet.append_wallet_blocks(wallet_blocks).unwrap(); + wallet + .extend_wallet_transactions(wallet_transactions) + .unwrap(); wallet.append_nullifiers(nullifiers).unwrap(); + // TODO: pararellise shard tree, this is currently the bottleneck on sync wallet.update_shard_trees(shard_tree_data).unwrap(); // TODO: add trait to save wallet data to persistance for in-memory wallets @@ -233,17 +239,25 @@ where fn remove_irrelevant_data(wallet: &mut W, scan_range: &ScanRange) -> Result<(), ()> where - W: SyncBlocks + SyncNullifiers, + W: SyncWallet + SyncBlocks + SyncNullifiers, { if scan_range.priority() != ScanPriority::Historic { return Ok(()); } - // TODO: also retain blocks that contain transactions relevant to the wallet - wallet - .get_wallet_blocks_mut() + let wallet_height = wallet + .get_sync_state() .unwrap() - .retain(|height, _| *height >= scan_range.block_range().end); + .scan_ranges() + .last() + .expect("wallet should always have scan ranges after sync has started") + .block_range() + .end; + + // TODO: also retain blocks that contain transactions relevant to the wallet + wallet.get_wallet_blocks_mut().unwrap().retain(|height, _| { + *height >= scan_range.block_range().end || *height >= wallet_height.saturating_sub(100) + }); wallet .get_nullifiers_mut() .unwrap() diff --git a/zingo-sync/src/interface.rs b/zingo-sync/src/traits.rs similarity index 63% rename from zingo-sync/src/interface.rs rename to zingo-sync/src/traits.rs index 6a1145074..64028bcca 100644 --- a/zingo-sync/src/interface.rs +++ b/zingo-sync/src/traits.rs @@ -5,9 +5,10 @@ use std::fmt::Debug; use zcash_client_backend::keys::UnifiedFullViewingKey; use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::AccountId; -use crate::primitives::{NullifierMap, SyncState, WalletBlock}; +use crate::primitives::{NullifierMap, SyncState, WalletBlock, WalletTransaction}; use crate::witness::{ShardTreeData, ShardTrees}; /// Temporary dump for all neccessary wallet functionality for PoC @@ -18,6 +19,9 @@ pub trait SyncWallet { /// Returns block height wallet was created fn get_birthday(&self) -> Result; + /// Returns reference to wallet sync state + fn get_sync_state(&self) -> Result<&SyncState, Self::Error>; + /// Returns mutable reference to wallet sync state fn get_sync_state_mut(&mut self) -> Result<&mut SyncState, Self::Error>; @@ -27,7 +31,7 @@ pub trait SyncWallet { ) -> Result, Self::Error>; } -/// Trait for interfacing sync engine [`crate::primitives::WalletBlock`] with wallet data +/// Trait for interfacing [`crate::primitives::WalletBlock`]s with wallet data pub trait SyncBlocks: SyncWallet { // TODO: add method to get wallet data for writing defualt implementations on other methods @@ -43,8 +47,31 @@ pub trait SyncBlocks: SyncWallet { /// Append wallet compact blocks to wallet data fn append_wallet_blocks( &mut self, - wallet_blocks: BTreeMap, - ) -> Result<(), Self::Error>; + mut wallet_blocks: BTreeMap, + ) -> Result<(), Self::Error> { + self.get_wallet_blocks_mut()?.append(&mut wallet_blocks); + + Ok(()) + } +} + +/// Trait for interfacing [`crate::primitives::WalletTransaction`]s with wallet data +pub trait SyncTransactions: SyncWallet { + /// Get mutable reference to wallet transactions + fn get_wallet_transactions_mut( + &mut self, + ) -> Result<&mut HashMap, Self::Error>; + + /// Extend wallet transaction map with new wallet transactions + fn extend_wallet_transactions( + &mut self, + wallet_transactions: HashMap, + ) -> Result<(), Self::Error> { + self.get_wallet_transactions_mut()? + .extend(wallet_transactions); + + Ok(()) + } } /// Trait for interfacing nullifiers with wallet data @@ -54,8 +81,17 @@ pub trait SyncNullifiers: SyncWallet { /// Get mutable reference to wallet nullifier map fn get_nullifiers_mut(&mut self) -> Result<&mut NullifierMap, Self::Error>; - /// Append nullifiers to wallet data - fn append_nullifiers(&mut self, nullifier_map: NullifierMap) -> Result<(), Self::Error>; + /// Append nullifiers to wallet nullifier map + fn append_nullifiers(&mut self, mut nullifier_map: NullifierMap) -> Result<(), Self::Error> { + self.get_nullifiers_mut()? + .sapling_mut() + .append(nullifier_map.sapling_mut()); + self.get_nullifiers_mut()? + .orchard_mut() + .append(nullifier_map.orchard_mut()); + + Ok(()) + } } /// Trait for interfacing shard tree data with wallet data @@ -63,7 +99,7 @@ pub trait SyncShardTrees: SyncWallet { /// Get mutable reference to shard trees fn get_shard_trees_mut(&mut self) -> Result<&mut ShardTrees, Self::Error>; - /// Update wallet data with shard tree data + /// Update wallet shard trees with new shard tree data fn update_shard_trees(&mut self, shard_tree_data: ShardTreeData) -> Result<(), Self::Error> { let ShardTreeData { sapling_initial_position, @@ -72,16 +108,14 @@ pub trait SyncShardTrees: SyncWallet { orchard_leaves_and_retentions, } = shard_tree_data; - self.get_shard_trees_mut() - .unwrap() + self.get_shard_trees_mut()? .sapling_mut() .batch_insert( sapling_initial_position, sapling_leaves_and_retentions.into_iter(), ) .unwrap(); - self.get_shard_trees_mut() - .unwrap() + self.get_shard_trees_mut()? .orchard_mut() .batch_insert( orchard_initial_position, diff --git a/zingolib/src/wallet.rs b/zingolib/src/wallet.rs index 76faeb6b2..57532ed8c 100644 --- a/zingolib/src/wallet.rs +++ b/zingolib/src/wallet.rs @@ -16,7 +16,7 @@ use rand::Rng; #[cfg(feature = "sync")] use zingo_sync::{ - primitives::{NullifierMap, SyncState, WalletBlock}, + primitives::{NullifierMap, SyncState, WalletBlock, WalletTransaction}, witness::ShardTrees, }; @@ -25,6 +25,7 @@ use bip0039::Mnemonic; use std::collections::BTreeMap; use std::{ cmp, + collections::HashMap, io::{self, Error, ErrorKind, Read, Write}, sync::{atomic::AtomicU64, Arc}, time::SystemTime, @@ -222,6 +223,11 @@ pub struct LightWallet { #[getset(get = "pub", get_mut = "pub")] wallet_blocks: BTreeMap, + /// Wallet transactions + #[cfg(feature = "sync")] + #[getset(get = "pub", get_mut = "pub")] + wallet_transactions: HashMap, + /// Nullifier map #[cfg(feature = "sync")] #[getset(get = "pub", get_mut = "pub")] @@ -410,6 +416,8 @@ impl LightWallet { #[cfg(feature = "sync")] wallet_blocks: BTreeMap::new(), #[cfg(feature = "sync")] + wallet_transactions: HashMap::new(), + #[cfg(feature = "sync")] nullifier_map: zingo_sync::primitives::NullifierMap::new(), #[cfg(feature = "sync")] shard_trees: zingo_sync::witness::ShardTrees::new(), diff --git a/zingolib/src/wallet/disk.rs b/zingolib/src/wallet/disk.rs index 83ec456bc..d84578fe0 100644 --- a/zingolib/src/wallet/disk.rs +++ b/zingolib/src/wallet/disk.rs @@ -6,6 +6,7 @@ use zcash_keys::keys::UnifiedSpendingKey; use zip32::AccountId; use std::{ + collections::HashMap, io::{self, Error, ErrorKind, Read, Write}, sync::{atomic::AtomicU64, Arc}, }; @@ -293,6 +294,8 @@ impl LightWallet { #[cfg(feature = "sync")] wallet_blocks: BTreeMap::new(), #[cfg(feature = "sync")] + wallet_transactions: HashMap::new(), + #[cfg(feature = "sync")] nullifier_map: zingo_sync::primitives::NullifierMap::new(), #[cfg(feature = "sync")] shard_trees: zingo_sync::witness::ShardTrees::new(), diff --git a/zingolib/src/wallet/sync.rs b/zingolib/src/wallet/sync.rs index b4b8be5eb..8496caef9 100644 --- a/zingolib/src/wallet/sync.rs +++ b/zingolib/src/wallet/sync.rs @@ -8,8 +8,8 @@ use std::{ use zcash_keys::keys::{UnifiedFullViewingKey, UnifiedSpendingKey}; use zcash_primitives::consensus::BlockHeight; use zingo_sync::{ - interface::{SyncBlocks, SyncNullifiers, SyncShardTrees, SyncWallet}, primitives::{NullifierMap, SyncState, WalletBlock}, + traits::{SyncBlocks, SyncNullifiers, SyncShardTrees, SyncTransactions, SyncWallet}, witness::ShardTrees, }; use zip32::AccountId; @@ -24,6 +24,10 @@ impl SyncWallet for LightWallet { Ok(BlockHeight::from_u32(birthday as u32)) } + fn get_sync_state(&self) -> Result<&SyncState, Self::Error> { + Ok(self.sync_state()) + } + fn get_sync_state_mut(&mut self) -> Result<&mut SyncState, Self::Error> { Ok(self.sync_state_mut()) } @@ -61,14 +65,19 @@ impl SyncBlocks for LightWallet { ) -> Result<&mut BTreeMap, Self::Error> { Ok(self.wallet_blocks_mut()) } +} - fn append_wallet_blocks( +impl SyncTransactions for LightWallet { + fn get_wallet_transactions_mut( &mut self, - mut wallet_compact_blocks: BTreeMap, - ) -> Result<(), Self::Error> { - self.wallet_blocks.append(&mut wallet_compact_blocks); - - Ok(()) + ) -> Result< + &mut HashMap< + zcash_primitives::transaction::TxId, + zingo_sync::primitives::WalletTransaction, + >, + Self::Error, + > { + Ok(self.wallet_transactions_mut()) } } @@ -76,17 +85,6 @@ impl SyncNullifiers for LightWallet { fn get_nullifiers_mut(&mut self) -> Result<&mut NullifierMap, ()> { Ok(self.nullifier_map_mut()) } - - fn append_nullifiers(&mut self, mut nullifier_map: NullifierMap) -> Result<(), Self::Error> { - self.nullifier_map - .sapling_mut() - .append(nullifier_map.sapling_mut()); - self.nullifier_map - .orchard_mut() - .append(nullifier_map.orchard_mut()); - - Ok(()) - } } impl SyncShardTrees for LightWallet {