From 651310ce2a36e176bba20ebe6bfa83f85626f97e Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 10 Apr 2024 15:57:52 +0200
Subject: [PATCH 01/67] use yellowstone-proto v1.13.0+solana.1.17.28

---
 Cargo.lock | 831 ++++++++++++++++++++++++++---------------------------
 Cargo.toml |   8 +-
 2 files changed, 404 insertions(+), 435 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ec59b1f..6dc31a7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -65,23 +65,23 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.7.7"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd"
+checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
 dependencies = [
- "getrandom 0.2.11",
+ "getrandom 0.2.14",
  "once_cell",
  "version_check",
 ]
 
 [[package]]
 name = "ahash"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72832d73be48bac96a5d7944568f305d829ed55b0ce3b483647089dfaf6cf704"
+checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d"
 dependencies = [
  "cfg-if",
- "getrandom 0.2.11",
+ "getrandom 0.2.14",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -89,9 +89,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.1.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
 dependencies = [
  "memchr",
 ]
@@ -128,9 +128,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.75"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
 
 [[package]]
 name = "ark-bn254"
@@ -275,9 +275,9 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9"
 
 [[package]]
 name = "async-compression"
-version = "0.4.5"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5"
+checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60"
 dependencies = [
  "brotli",
  "flate2",
@@ -306,18 +306,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.74"
+version = "0.1.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
+checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -333,15 +333,15 @@ dependencies = [
 
 [[package]]
 name = "autocfg"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80"
 
 [[package]]
 name = "autotools"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77"
+checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf"
 dependencies = [
  "cc",
 ]
@@ -393,9 +393,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.69"
+version = "0.3.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
+checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d"
 dependencies = [
  "addr2line",
  "cc",
@@ -414,9 +414,9 @@ checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
 
 [[package]]
 name = "base64"
-version = "0.21.5"
+version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "bincode"
@@ -435,9 +435,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
 dependencies = [
  "serde",
 ]
@@ -453,9 +453,9 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87"
+checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -582,9 +582,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "3.4.0"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f"
+checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -593,9 +593,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "2.5.1"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
+checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -609,9 +609,9 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3"
 
 [[package]]
 name = "bumpalo"
-version = "3.14.0"
+version = "3.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
+checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
 
 [[package]]
 name = "bv"
@@ -625,22 +625,22 @@ dependencies = [
 
 [[package]]
 name = "bytemuck"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
+checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15"
 dependencies = [
  "bytemuck_derive",
 ]
 
 [[package]]
 name = "bytemuck_derive"
-version = "1.5.0"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1"
+checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -651,15 +651,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.5.0"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 
 [[package]]
 name = "cc"
-version = "1.0.83"
+version = "1.0.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41"
 dependencies = [
  "jobserver",
  "libc",
@@ -673,9 +673,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.31"
+version = "0.4.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -683,7 +683,7 @@ dependencies = [
  "num-traits",
  "serde",
  "wasm-bindgen",
- "windows-targets 0.48.5",
+ "windows-targets 0.52.4",
 ]
 
 [[package]]
@@ -758,63 +758,55 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
+checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "crc32fast"
-version = "1.3.2"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.9"
+version = "0.5.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5"
+checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95"
 dependencies = [
- "cfg-if",
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
 dependencies = [
- "cfg-if",
  "crossbeam-epoch",
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.16"
+version = "0.9.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
 dependencies = [
- "autocfg",
- "cfg-if",
  "crossbeam-utils",
- "memoffset",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.17"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f"
-dependencies = [
- "cfg-if",
-]
+checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 
 [[package]]
 name = "crunchy"
@@ -888,9 +880,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.20.3"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
+checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -898,27 +890,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.3"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
+checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.20.3"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
+checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -1014,37 +1006,37 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.33"
+version = "0.8.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
+checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "enum-iterator"
-version = "1.4.1"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689"
+checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94"
 dependencies = [
  "enum-iterator-derive",
 ]
 
 [[package]]
 name = "enum-iterator-derive"
-version = "1.2.1"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb"
+checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -1078,9 +1070,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.0.1"
+version = "2.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984"
 
 [[package]]
 name = "feature-probe"
@@ -1121,9 +1113,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
+checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1136,9 +1128,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
+checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -1146,15 +1138,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
+checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
+checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -1163,38 +1155,38 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "futures-sink"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
+checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
 
 [[package]]
 name = "futures-task"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
+checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
 
 [[package]]
 name = "futures-util"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
+checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1244,9 +1236,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.11"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
+checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -1261,7 +1253,7 @@ version = "0.10.1+yellowstone.1.12"
 dependencies = [
  "anyhow",
  "async-stream",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
  "csv",
  "derive_more",
@@ -1298,9 +1290,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.22"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
+checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
 dependencies = [
  "bytes",
  "fnv",
@@ -1308,7 +1300,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap 2.1.0",
+ "indexmap 2.2.6",
  "slab",
  "tokio",
  "tokio-util",
@@ -1330,7 +1322,7 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
- "ahash 0.7.7",
+ "ahash 0.7.8",
 ]
 
 [[package]]
@@ -1345,7 +1337,7 @@ version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
 dependencies = [
- "ahash 0.8.4",
+ "ahash 0.8.5",
 ]
 
 [[package]]
@@ -1356,9 +1348,9 @@ checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 
 [[package]]
 name = "heck"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
@@ -1371,9 +1363,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.3"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
 
 [[package]]
 name = "hmac"
@@ -1405,20 +1397,11 @@ dependencies = [
  "hmac 0.8.1",
 ]
 
-[[package]]
-name = "home"
-version = "0.5.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
-dependencies = [
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "http"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
 dependencies = [
  "bytes",
  "fnv",
@@ -1506,9 +1489,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.58"
+version = "0.1.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
+checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -1571,9 +1554,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.1.0"
+version = "2.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
+checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.3",
@@ -1596,42 +1579,42 @@ dependencies = [
 
 [[package]]
 name = "itertools"
-version = "0.11.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
 dependencies = [
  "either",
 ]
 
 [[package]]
 name = "itoa"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
 [[package]]
 name = "jobserver"
-version = "0.1.27"
+version = "0.1.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
+checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.66"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca"
+checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
 dependencies = [
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "keccak"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940"
+checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654"
 dependencies = [
  "cpufeatures",
 ]
@@ -1644,9 +1627,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.151"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "libsecp256k1"
@@ -1710,9 +1693,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
+checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
 
 [[package]]
 name = "lock_api"
@@ -1726,9 +1709,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
 
 [[package]]
 name = "matchit"
@@ -1738,9 +1721,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
 [[package]]
 name = "memchr"
-version = "2.6.4"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
+checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
 
 [[package]]
 name = "memmap2"
@@ -1753,9 +1736,9 @@ dependencies = [
 
 [[package]]
 name = "memoffset"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
 dependencies = [
  "autocfg",
 ]
@@ -1790,18 +1773,18 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
 dependencies = [
  "adler",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
 dependencies = [
  "libc",
  "wasi 0.11.0+wasi-snapshot-preview1",
@@ -1810,9 +1793,9 @@ dependencies = [
 
 [[package]]
 name = "multimap"
-version = "0.8.3"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
 
 [[package]]
 name = "nu-ansi-term"
@@ -1883,30 +1866,29 @@ dependencies = [
 
 [[package]]
 name = "num-derive"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712"
+checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "num-integer"
-version = "0.1.45"
+version = "0.1.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
 dependencies = [
- "autocfg",
  "num-traits",
 ]
 
 [[package]]
 name = "num-iter"
-version = "0.1.43"
+version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
+checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9"
 dependencies = [
  "autocfg",
  "num-integer",
@@ -1927,9 +1909,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.17"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
+checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
 dependencies = [
  "autocfg",
 ]
@@ -1940,7 +1922,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.3.3",
+ "hermit-abi 0.3.9",
  "libc",
 ]
 
@@ -1955,11 +1937,11 @@ dependencies = [
 
 [[package]]
 name = "num_enum"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683751d591e6d81200c39fb0d1032608b77724f34114db54f571ff1317b337c0"
+checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"
 dependencies = [
- "num_enum_derive 0.7.1",
+ "num_enum_derive 0.7.2",
 ]
 
 [[package]]
@@ -1971,26 +1953,26 @@ dependencies = [
  "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "num_enum_derive"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e"
+checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
 dependencies = [
- "proc-macro-crate 2.0.1",
+ "proc-macro-crate 3.1.0",
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "object"
-version = "0.32.1"
+version = "0.32.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
+checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
 dependencies = [
  "memchr",
 ]
@@ -2003,9 +1985,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "opaque-debug"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
 [[package]]
 name = "openssl-probe"
@@ -2088,34 +2070,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap 2.1.0",
+ "indexmap 2.2.6",
 ]
 
 [[package]]
 name = "pin-project"
-version = "1.1.3"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.3"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
 
 [[package]]
 name = "pin-utils"
@@ -2125,9 +2107,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.27"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
+checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
 [[package]]
 name = "plain"
@@ -2155,12 +2137,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "prettyplease"
-version = "0.2.15"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
+checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7"
 dependencies = [
  "proc-macro2",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -2184,28 +2166,27 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "2.0.1"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97dc5fea232fc28d2f597b37c4876b348a40e33f3b02cc975c8d006d78d94b1a"
+checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
 dependencies = [
- "toml_datetime",
- "toml_edit 0.20.2",
+ "toml_edit 0.21.1",
 ]
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.70"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
+checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "prost"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
+checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -2213,13 +2194,13 @@ dependencies = [
 
 [[package]]
 name = "prost-build"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2"
+checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1"
 dependencies = [
  "bytes",
  "heck",
- "itertools 0.11.0",
+ "itertools 0.12.1",
  "log",
  "multimap",
  "once_cell",
@@ -2228,29 +2209,28 @@ dependencies = [
  "prost",
  "prost-types",
  "regex",
- "syn 2.0.41",
+ "syn 2.0.58",
  "tempfile",
- "which",
 ]
 
 [[package]]
 name = "prost-derive"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
+checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48"
 dependencies = [
  "anyhow",
- "itertools 0.11.0",
+ "itertools 0.12.1",
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "prost-types"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"
+checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe"
 dependencies = [
  "prost",
 ]
@@ -2281,14 +2261,14 @@ checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
 dependencies = [
  "proc-macro2",
 ]
@@ -2352,7 +2332,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.11",
+ "getrandom 0.2.14",
 ]
 
 [[package]]
@@ -2375,9 +2355,9 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
 dependencies = [
  "either",
  "rayon-core",
@@ -2385,9 +2365,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
 dependencies = [
  "crossbeam-deque",
  "crossbeam-utils",
@@ -2404,9 +2384,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.2"
+version = "1.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2416,9 +2396,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.3"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2427,18 +2407,18 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
 
 [[package]]
 name = "reqwest"
-version = "0.11.23"
+version = "0.11.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41"
+checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
 dependencies = [
  "async-compression",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -2460,6 +2440,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded",
+ "sync_wrapper",
  "system-configuration",
  "tokio",
  "tokio-rustls",
@@ -2475,16 +2456,17 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.7"
+version = "0.17.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
- "getrandom 0.2.11",
+ "cfg-if",
+ "getrandom 0.2.14",
  "libc",
  "spin",
  "untrusted",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -2510,11 +2492,11 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.28"
+version = "0.38.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
+checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags 2.5.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -2551,7 +2533,7 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
 dependencies = [
- "base64 0.21.5",
+ "base64 0.21.7",
 ]
 
 [[package]]
@@ -2566,23 +2548,23 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.14"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
+checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47"
 
 [[package]]
 name = "ryu"
-version = "1.0.16"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
+checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
 
 [[package]]
 name = "schannel"
-version = "0.1.22"
+version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -2608,7 +2590,7 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -2623,9 +2605,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.9.2"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
+checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6"
 dependencies = [
  "bitflags 1.3.2",
  "core-foundation",
@@ -2636,9 +2618,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.9.1"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
+checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -2646,44 +2628,44 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.20"
+version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
 
 [[package]]
 name = "serde"
-version = "1.0.193"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
+checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_bytes"
-version = "0.11.12"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff"
+checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.193"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
+checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.108"
+version = "1.0.115"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
+checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd"
 dependencies = [
  "itoa",
  "ryu",
@@ -2721,7 +2703,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -2806,28 +2788,28 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.11.2"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
 
 [[package]]
 name = "socket2"
-version = "0.5.5"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
+checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871"
 dependencies = [
  "libc",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "solana-account-decoder"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ea4bedfcc8686ae6d01a3d8288f5b9746cd00ec63f0ce9a6415849d35add50"
+checksum = "8d76c43ef61f527d719b5c6bfa5a62ebba60839739125da9e8a00fb82349afd2"
 dependencies = [
  "Inflector",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
  "bs58",
  "bv",
@@ -2847,9 +2829,9 @@ dependencies = [
 
 [[package]]
 name = "solana-config-program"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de23cd0dd8673f4590e90bfa47ff19eb629f4b7dc15a3fb173a62d932801d07"
+checksum = "e3afd4e309d304e296765cab716fb1fd66c66ec300465c8b26f8cce763275132"
 dependencies = [
  "bincode",
  "chrono",
@@ -2861,11 +2843,11 @@ dependencies = [
 
 [[package]]
 name = "solana-frozen-abi"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4090f2ac64149ce1fbabd5277f41e278edc1f38121927fe8f6355e67ead3e199"
+checksum = "fb1b8230474ae9f7c841060c299999124582e8d2a0448d7847720792e98cc64e"
 dependencies = [
- "ahash 0.8.4",
+ "ahash 0.8.5",
  "blake3",
  "block-buffer 0.10.4",
  "bs58",
@@ -2891,21 +2873,21 @@ dependencies = [
 
 [[package]]
 name = "solana-frozen-abi-macro"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "765bcdc1ecc31ea5d3d7ddb680ffa6645809c122b4ffdc223b161850e6ba352b"
+checksum = "793910ab733b113b80c357f8f492dda2fabd5671c4ea03db3aa4e46b938fdbe3"
 dependencies = [
  "proc-macro2",
  "quote",
  "rustc_version",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "solana-logger"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7f3cad088bc5f00569cb5b4c3aaba8d935f8f7cc25c91cc0c55a8a7de2b137"
+checksum = "6d3f819af39632dc538a566c937253bf46256e4c0e60f621c6db448bc7c76294"
 dependencies = [
  "env_logger",
  "lazy_static",
@@ -2914,9 +2896,9 @@ dependencies = [
 
 [[package]]
 name = "solana-measure"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2de5041d16120852c0deea047c024e1fad8819e49041491f0cca6c91c243fd5d"
+checksum = "cb045f0235b16f7d926f6e0338db822747d61559a1368c3cb017ba6e02c516d0"
 dependencies = [
  "log",
  "solana-sdk",
@@ -2924,9 +2906,9 @@ dependencies = [
 
 [[package]]
 name = "solana-metrics"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fd6f25f0076b6eb873f7e2a85e53191ac2affe6782131be1a2867d057307e20"
+checksum = "1af84362ad5804dc64ca88b1ca5c35bd41321e12d42c798ac06a6fbb60dd0e70"
 dependencies = [
  "crossbeam-channel",
  "gethostname",
@@ -2939,17 +2921,17 @@ dependencies = [
 
 [[package]]
 name = "solana-program"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1141d1dffbe68852128f7bbcc3c43a5d2cb715ecffeeb64eb81bb93cbaf80bb"
+checksum = "581f38a870bffbe623d900c68579984671f8dfa35bbfb3309d7134de22ce8652"
 dependencies = [
  "ark-bn254",
  "ark-ec",
  "ark-ff",
  "ark-serialize",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
- "bitflags 2.4.1",
+ "bitflags 2.5.0",
  "blake3",
  "borsh 0.10.3",
  "borsh 0.9.3",
@@ -2960,7 +2942,7 @@ dependencies = [
  "console_error_panic_hook",
  "console_log",
  "curve25519-dalek",
- "getrandom 0.2.11",
+ "getrandom 0.2.14",
  "itertools 0.10.5",
  "js-sys",
  "lazy_static",
@@ -2993,11 +2975,11 @@ dependencies = [
 
 [[package]]
 name = "solana-program-runtime"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "942de577a2865cec28fc174575c9bd6cf7af815832af67fe40ca856075550998"
+checksum = "490b6f65aced077e0c5e57c20f151a134458fc350905c20d7dcf3f2162eaa6f6"
 dependencies = [
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
  "eager",
  "enum-iterator",
@@ -3021,14 +3003,14 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "278a95acb99377dd4585599fdbec23d0a6fcb94ec78285283723fdd365fe885e"
+checksum = "0d70ab837cc79ed67df6fdb145f1ffd544f1eaa60b0757b750f4864b90498bad"
 dependencies = [
  "assert_matches",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
- "bitflags 2.4.1",
+ "bitflags 2.5.0",
  "borsh 0.10.3",
  "bs58",
  "bytemuck",
@@ -3075,15 +3057,15 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-macro"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92dbaf563210f61828800f2a3d8c188fa2afede91920d364982e280318db2eb5"
+checksum = "5f9d0433c4084a3260a32ec67f6b4272c4232d15e732be542cd5dfdf0ae1e784"
 dependencies = [
  "bs58",
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -3094,12 +3076,12 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183"
 
 [[package]]
 name = "solana-transaction-status"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e2031070cba17802f7108b53f6db01b82cdfb0360b0a8b9d51c584f2e9dd9e4"
+checksum = "29f58f2f864d900eddf2e21a99ebe445b6be525d597e44952f075d8237035b8e"
 dependencies = [
  "Inflector",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
  "borsh 0.10.3",
  "bs58",
@@ -3119,12 +3101,12 @@ dependencies = [
 
 [[package]]
 name = "solana-zk-token-sdk"
-version = "1.17.15"
+version = "1.17.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef26fb44734aa940e6648bbbeead677edc68c7e1ec09128e5f16a8924c389a38"
+checksum = "5aef1b48d9fdb2619349d2d15942d83c99aabe995ff945d9b418176373aa823c"
 dependencies = [
  "aes-gcm-siv",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bincode",
  "bytemuck",
  "byteorder",
@@ -3179,7 +3161,7 @@ checksum = "992d9c64c2564cc8f63a4b508bf3ebcdf2254b0429b13cd1d31adb6162432a5f"
 dependencies = [
  "assert_matches",
  "borsh 0.10.3",
- "num-derive 0.4.1",
+ "num-derive 0.4.2",
  "num-traits",
  "solana-program",
  "spl-token",
@@ -3189,9 +3171,9 @@ dependencies = [
 
 [[package]]
 name = "spl-discriminator"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cce5d563b58ef1bb2cdbbfe0dfb9ffdc24903b10ae6a4df2d8f425ece375033f"
+checksum = "daa600f2fe56f32e923261719bae640d873edadbc5237681a39b8e37bfd4d263"
 dependencies = [
  "bytemuck",
  "solana-program",
@@ -3200,25 +3182,25 @@ dependencies = [
 
 [[package]]
 name = "spl-discriminator-derive"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fadbefec4f3c678215ca72bd71862697bb06b41fd77c0088902dd3203354387b"
+checksum = "07fd7858fc4ff8fb0e34090e41d7eb06a823e1057945c26d480bfc21d2338a93"
 dependencies = [
  "quote",
  "spl-discriminator-syn",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "spl-discriminator-syn"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e5f2044ca42c8938d54d1255ce599c79a1ffd86b677dfab695caa20f9ffc3f2"
+checksum = "18fea7be851bd98d10721782ea958097c03a0c2a07d8d4997041d0ece6319a63"
 dependencies = [
  "proc-macro2",
  "quote",
  "sha2 0.10.8",
- "syn 2.0.41",
+ "syn 2.0.58",
  "thiserror",
 ]
 
@@ -3233,9 +3215,9 @@ dependencies = [
 
 [[package]]
 name = "spl-pod"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2881dddfca792737c0706fa0175345ab282b1b0879c7d877bad129645737c079"
+checksum = "85a5db7e4efb1107b0b8e52a13f035437cdcb36ef99c58f6d467f089d9b2915a"
 dependencies = [
  "borsh 0.10.3",
  "bytemuck",
@@ -3246,11 +3228,11 @@ dependencies = [
 
 [[package]]
 name = "spl-program-error"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "249e0318493b6bcf27ae9902600566c689b7dfba9f1bdff5893e92253374e78c"
+checksum = "7e0657b6490196971d9e729520ba934911ff41fbb2cb9004463dbe23cf8b4b4f"
 dependencies = [
- "num-derive 0.4.1",
+ "num-derive 0.4.2",
  "num-traits",
  "solana-program",
  "spl-program-error-derive",
@@ -3259,21 +3241,21 @@ dependencies = [
 
 [[package]]
 name = "spl-program-error-derive"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5269c8e868da17b6552ef35a51355a017bd8e0eae269c201fef830d35fa52c"
+checksum = "1845dfe71fd68f70382232742e758557afe973ae19e6c06807b2c30f5d5cb474"
 dependencies = [
  "proc-macro2",
  "quote",
  "sha2 0.10.8",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "spl-tlv-account-resolution"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "615d381f48ddd2bb3c57c7f7fb207591a2a05054639b18a62e785117dd7a8683"
+checksum = "56f335787add7fa711819f9e7c573f8145a5358a709446fe2d24bf2a88117c90"
 dependencies = [
  "bytemuck",
  "solana-program",
@@ -3306,9 +3288,9 @@ checksum = "d697fac19fd74ff472dfcc13f0b442dd71403178ce1de7b5d16f83a33561c059"
 dependencies = [
  "arrayref",
  "bytemuck",
- "num-derive 0.4.1",
+ "num-derive 0.4.2",
  "num-traits",
- "num_enum 0.7.1",
+ "num_enum 0.7.2",
  "solana-program",
  "solana-security-txt",
  "solana-zk-token-sdk",
@@ -3367,9 +3349,9 @@ dependencies = [
 
 [[package]]
 name = "spl-type-length-value"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a468e6f6371f9c69aae760186ea9f1a01c2908351b06a5e0026d21cfc4d7ecac"
+checksum = "8f9ebd75d29c5f48de5f6a9c114e08531030b75b8ac2c557600ac7da0b73b1e8"
 dependencies = [
  "bytemuck",
  "solana-program",
@@ -3403,9 +3385,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.41"
+version = "2.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269"
+checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3441,51 +3423,50 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.8.1"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
+checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall",
  "rustix",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "termcolor"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
 dependencies = [
  "winapi-util",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.51"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7"
+checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.51"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df"
+checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
 name = "thread_local"
-version = "1.1.7"
+version = "1.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
+checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -3527,9 +3508,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.35.1"
+version = "1.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104"
+checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787"
 dependencies = [
  "backtrace",
  "bytes",
@@ -3560,7 +3541,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -3575,9 +3556,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.14"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
+checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -3609,9 +3590,9 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.3"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
+checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
 
 [[package]]
 name = "toml_edit"
@@ -3619,18 +3600,18 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap 2.1.0",
+ "indexmap 2.2.6",
  "toml_datetime",
  "winnow",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.20.2"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
+checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
 dependencies = [
- "indexmap 2.1.0",
+ "indexmap 2.2.6",
  "toml_datetime",
  "winnow",
 ]
@@ -3644,7 +3625,7 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum",
- "base64 0.21.5",
+ "base64 0.21.7",
  "bytes",
  "h2",
  "http",
@@ -3676,7 +3657,7 @@ dependencies = [
  "proc-macro2",
  "prost-build",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -3743,7 +3724,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -3795,9 +3776,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.14"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416"
+checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
 
 [[package]]
 name = "unicode-ident"
@@ -3807,9 +3788,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.22"
+version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
+checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
 dependencies = [
  "tinyvec",
 ]
@@ -3901,9 +3882,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.89"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e"
+checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -3911,24 +3892,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.89"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826"
+checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.39"
+version = "0.4.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12"
+checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -3938,9 +3919,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.89"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2"
+checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -3948,28 +3929,28 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.89"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
+checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.89"
+version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f"
+checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
 
 [[package]]
 name = "web-sys"
-version = "0.3.66"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f"
+checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3977,21 +3958,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.25.3"
+version = "0.25.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
-
-[[package]]
-name = "which"
-version = "4.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
-dependencies = [
- "either",
- "home",
- "once_cell",
- "rustix",
-]
+checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
 [[package]]
 name = "winapi"
@@ -4026,11 +3995,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-core"
-version = "0.51.1"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
 dependencies = [
- "windows-targets 0.48.5",
+ "windows-targets 0.52.4",
 ]
 
 [[package]]
@@ -4048,7 +4017,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.0",
+ "windows-targets 0.52.4",
 ]
 
 [[package]]
@@ -4068,17 +4037,17 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
+checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.0",
- "windows_aarch64_msvc 0.52.0",
- "windows_i686_gnu 0.52.0",
- "windows_i686_msvc 0.52.0",
- "windows_x86_64_gnu 0.52.0",
- "windows_x86_64_gnullvm 0.52.0",
- "windows_x86_64_msvc 0.52.0",
+ "windows_aarch64_gnullvm 0.52.4",
+ "windows_aarch64_msvc 0.52.4",
+ "windows_i686_gnu 0.52.4",
+ "windows_i686_msvc 0.52.4",
+ "windows_x86_64_gnu 0.52.4",
+ "windows_x86_64_gnullvm 0.52.4",
+ "windows_x86_64_msvc 0.52.4",
 ]
 
 [[package]]
@@ -4089,9 +4058,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
+checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -4101,9 +4070,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
+checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -4113,9 +4082,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
+checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -4125,9 +4094,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
+checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -4137,9 +4106,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
+checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -4149,9 +4118,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
+checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -4161,15 +4130,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.0"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
+checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
 
 [[package]]
 name = "winnow"
-version = "0.5.30"
+version = "0.5.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5"
+checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
 dependencies = [
  "memchr",
 ]
@@ -4186,8 +4155,8 @@ dependencies = [
 
 [[package]]
 name = "yellowstone-grpc-client"
-version = "1.13.0+solana.1.17.15"
-source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.12.0+solana.1.17.15#c7b72cc8781c2dc48e4a7c94e411f95df495cf2f"
+version = "1.14.0+solana.1.17.28"
+source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.13.0+solana.1.17.28#aafd827df872f093535c425ded4eca423929a3d0"
 dependencies = [
  "bytes",
  "futures",
@@ -4200,8 +4169,8 @@ dependencies = [
 
 [[package]]
 name = "yellowstone-grpc-proto"
-version = "1.12.0+solana.1.17.15"
-source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.12.0+solana.1.17.15#c7b72cc8781c2dc48e4a7c94e411f95df495cf2f"
+version = "1.13.0+solana.1.17.28"
+source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.13.0+solana.1.17.28#aafd827df872f093535c425ded4eca423929a3d0"
 dependencies = [
  "anyhow",
  "bincode",
@@ -4216,22 +4185,22 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.7.31"
+version = "0.7.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d"
+checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.7.31"
+version = "0.7.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a"
+checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -4251,7 +4220,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -4275,9 +4244,9 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.9+zstd.1.5.5"
+version = "2.0.10+zstd.1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
+checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
 dependencies = [
  "cc",
  "pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index 1d4ece6..418905d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,16 +9,16 @@ authors = ["GroovieGermanikus <groovie@mango.markets>"]
 repository = "https://github.com/blockworks-foundation/geyser-grpc-connector"
 
 [dependencies]
-yellowstone-grpc-client = { version = "1.13.0+solana.1.17.15", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" }
-yellowstone-grpc-proto = { version = "1.12.0+solana.1.17.15", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" }
+yellowstone-grpc-client = { version = "1.14.0", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.13.0+solana.1.17.28" }
+yellowstone-grpc-proto = { version = "1.13.0", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.13.0+solana.1.17.28" }
 
 
 # required for CommitmentConfig
-solana-sdk = "~1.17.15"
+solana-sdk = "1"
 
 url = "2.5.0"
 async-stream = "0.3.5"
-tokio = { version = "1.28" , features = ["rt"] }
+tokio = { version = "1.28" , features = ["rt", "rt-multi-thread"] }
 futures = "0.3.28"
 merge-streams = "0.1.2"
 anyhow = "1.0.70"

From c1ca3e2e67b9ff55d324a448416af3e00fc1ae28 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 11 Apr 2024 14:57:04 +0200
Subject: [PATCH 02/67] 1.13 with buffer window size

---
 Cargo.lock                                   |   1 +
 Cargo.toml                                   |   2 +
 examples/stream_blocks_single.rs             |  11 +-
 src/grpc_subscription_autoreconnect_tasks.rs |   6 +-
 src/lib.rs                                   |   1 +
 src/yellowstone_grpc_util.rs                 | 113 +++++++++++++++++++
 6 files changed, 127 insertions(+), 7 deletions(-)
 create mode 100644 src/yellowstone_grpc_util.rs

diff --git a/Cargo.lock b/Cargo.lock
index 6dc31a7..1864a0e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1264,6 +1264,7 @@ dependencies = [
  "solana-logger",
  "solana-sdk",
  "tokio",
+ "tonic-health",
  "tracing",
  "tracing-subscriber",
  "url",
diff --git a/Cargo.toml b/Cargo.toml
index 418905d..5a49454 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,6 +32,8 @@ bincode = "1.3.3"
 
 csv = "1.3.0"
 
+tonic-health = "0.10.2"
+
 [dev-dependencies]
 tracing-subscriber = "0.3.16"
 solana-logger = "1"
diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs
index bff3b4e..e189149 100644
--- a/examples/stream_blocks_single.rs
+++ b/examples/stream_blocks_single.rs
@@ -84,10 +84,10 @@ pub async fn main() {
     );
 
     let timeouts = GrpcConnectionTimeouts {
-        connect_timeout: Duration::from_secs(5),
-        request_timeout: Duration::from_secs(5),
-        subscribe_timeout: Duration::from_secs(5),
-        receive_timeout: Duration::from_secs(5),
+        connect_timeout: Duration::from_secs(25),
+        request_timeout: Duration::from_secs(25),
+        subscribe_timeout: Duration::from_secs(25),
+        receive_timeout: Duration::from_secs(25),
     };
 
     let config = GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
@@ -137,9 +137,10 @@ pub async fn main() {
         while let Some(message) = blue_stream.next().await {
             match message {
                 Message::GeyserSubscribeUpdate(subscriber_update) => {
+                    let bytes = subscriber_update.encoded_len();
                     let mapped = map_block_update(*subscriber_update);
                     if let Some(slot) = mapped {
-                        info!("got update (blue)!!! slot: {}", slot);
+                        info!("got update (blue)!!! slot: {}, {} bytes", slot, bytes);
                     }
                 }
                 Message::Connecting(attempt) => {
diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index 4413b97..9e47887 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -10,6 +10,7 @@ use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError};
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::service::Interceptor;
 use yellowstone_grpc_proto::tonic::Status;
+use crate::yellowstone_grpc_util::{connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig};
 
 enum ConnectionState<S: Stream<Item = Result<SubscribeUpdate, Status>>, F: Interceptor> {
     NotConnected(Attempt),
@@ -75,13 +76,14 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         attempt,
                         addr
                     );
-                    let connect_result = GeyserGrpcClient::connect_with_timeout(
+
+                    let connect_result = connect_with_timeout_with_buffers(
                         addr,
                         token,
                         config,
                         connect_timeout,
                         request_timeout,
-                        false,
+                        GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter),
                     )
                     .await;
 
diff --git a/src/lib.rs b/src/lib.rs
index 64edd90..1e13e55 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -9,6 +9,7 @@ pub mod channel_plugger;
 pub mod grpc_subscription_autoreconnect_streams;
 pub mod grpc_subscription_autoreconnect_tasks;
 pub mod grpcmultiplex_fastestwins;
+mod yellowstone_grpc_util;
 mod obfuscate;
 
 // 1-based attempt counter
diff --git a/src/yellowstone_grpc_util.rs b/src/yellowstone_grpc_util.rs
new file mode 100644
index 0000000..bb5d4b9
--- /dev/null
+++ b/src/yellowstone_grpc_util.rs
@@ -0,0 +1,113 @@
+use std::time::Duration;
+use tonic_health::pb::health_client::HealthClient;
+use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult, InterceptorXToken};
+use yellowstone_grpc_proto::geyser::geyser_client::GeyserClient;
+use yellowstone_grpc_proto::geyser::SubscribeRequest;
+use yellowstone_grpc_proto::prost::bytes::Bytes;
+use yellowstone_grpc_proto::tonic;
+use yellowstone_grpc_proto::tonic::metadata::AsciiMetadataValue;
+use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue;
+use yellowstone_grpc_proto::tonic::service::Interceptor;
+use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig;
+
+
+pub async fn connect_with_timeout<E, T>(
+    endpoint: E,
+    x_token: Option<T>,
+    tls_config: Option<ClientTlsConfig>,
+    connect_timeout: Option<Duration>,
+    request_timeout: Option<Duration>,
+    connect_lazy: bool,
+) -> GeyserGrpcClientResult<GeyserGrpcClient<impl Interceptor>>
+    where
+        E: Into<Bytes>,
+        T: TryInto<AsciiMetadataValue, Error = InvalidMetadataValue>,
+{
+    GeyserGrpcClient::connect_with_timeout(
+       endpoint, x_token, tls_config, connect_timeout, request_timeout, connect_lazy).await
+}
+
+
+// see https://github.com/hyperium/tonic/blob/v0.10.2/tonic/src/transport/channel/mod.rs
+const DEFAULT_BUFFER_SIZE: usize = 1024;
+// see https://github.com/hyperium/hyper/blob/v0.14.28/src/proto/h2/client.rs#L45
+const DEFAULT_CONN_WINDOW: u32 = 1024 * 1024 * 5; // 5mb
+const DEFAULT_STREAM_WINDOW: u32 = 1024 * 1024 * 2; // 2mb
+
+#[derive(Debug, Clone)]
+pub struct GeyserGrpcClientBufferConfig {
+    pub buffer_size: Option<usize>,
+    pub conn_window: Option<u32>,
+    pub stream_window: Option<u32>,
+}
+
+impl Default for GeyserGrpcClientBufferConfig {
+    fn default() -> Self {
+        GeyserGrpcClientBufferConfig {
+            buffer_size: Some(DEFAULT_BUFFER_SIZE),
+            conn_window: Some(DEFAULT_CONN_WINDOW),
+            stream_window: Some(DEFAULT_STREAM_WINDOW),
+        }
+    }
+}
+
+impl GeyserGrpcClientBufferConfig {
+
+    pub fn optimize_for_subscription(filter: &SubscribeRequest) -> GeyserGrpcClientBufferConfig {
+        if !filter.blocks.is_empty() {
+            GeyserGrpcClientBufferConfig {
+                buffer_size: Some(65536),  // 64kb (default: 1k)
+                conn_window: Some(5242880), // 5mb (=default)
+                stream_window: Some(4194304), // 4mb (default: 2m)
+            }
+        } else {
+            GeyserGrpcClientBufferConfig::default()
+        }
+    }
+
+}
+
+
+pub async fn connect_with_timeout_with_buffers<E, T>(
+    endpoint: E,
+    x_token: Option<T>,
+    tls_config: Option<ClientTlsConfig>,
+    connect_timeout: Option<Duration>,
+    request_timeout: Option<Duration>,
+    buffer_config: GeyserGrpcClientBufferConfig,
+) -> GeyserGrpcClientResult<GeyserGrpcClient<impl Interceptor>>
+    where
+        E: Into<Bytes>,
+        T: TryInto<AsciiMetadataValue, Error = InvalidMetadataValue>, {
+    // see https://github.com/blockworks-foundation/geyser-grpc-connector/issues/10
+    let mut endpoint = tonic::transport::Endpoint::from_shared(endpoint)?
+        .buffer_size(buffer_config.buffer_size)
+        .initial_connection_window_size(buffer_config.conn_window)
+        .initial_stream_window_size(buffer_config.stream_window);
+
+    if let Some(tls_config) = tls_config {
+        endpoint = endpoint.tls_config(tls_config)?;
+    }
+
+    if let Some(connect_timeout) = connect_timeout {
+        endpoint = endpoint.timeout(connect_timeout);
+    }
+
+    if let Some(request_timeout) = request_timeout {
+        endpoint = endpoint.timeout(request_timeout);
+    }
+
+    let x_token: Option<AsciiMetadataValue> = match x_token {
+        Some(x_token) => Some(x_token.try_into()?),
+        None => None,
+    };
+    let interceptor = InterceptorXToken { x_token };
+
+    let channel = endpoint.connect_lazy();
+    let mut client = GeyserGrpcClient::new(
+        HealthClient::with_interceptor(channel.clone(), interceptor.clone()),
+        GeyserClient::with_interceptor(channel, interceptor)
+            .max_decoding_message_size(GeyserGrpcClient::max_decoding_message_size()),
+    );
+    Ok(client)
+}
\ No newline at end of file

From 1d2937ad955f21ace1f2507a09733205aad16d3e Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 10:12:59 +0200
Subject: [PATCH 03/67] clippy

---
 examples/stream_blocks_mainnet_task.rs        |  6 ++----
 examples/stream_blocks_single.rs              |  1 -
 ...grpc_subscription_autoreconnect_streams.rs | 19 ++++++++++++-------
 src/lib.rs                                    |  2 +-
 src/yellowstone_grpc_util.rs                  |  2 +-
 5 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/examples/stream_blocks_mainnet_task.rs b/examples/stream_blocks_mainnet_task.rs
index eb3ea6e..431ae2e 100644
--- a/examples/stream_blocks_mainnet_task.rs
+++ b/examples/stream_blocks_mainnet_task.rs
@@ -1,9 +1,7 @@
-use futures::Stream;
 use log::{info, warn};
 use solana_sdk::clock::Slot;
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
-use std::pin::pin;
 
 use base64::Engine;
 use itertools::Itertools;
@@ -23,10 +21,10 @@ use tokio::sync::mpsc::Receiver;
 use yellowstone_grpc_proto::geyser::SubscribeUpdateBlock;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{
-    create_geyser_autoconnection_task, create_geyser_autoconnection_task_with_mpsc,
+    create_geyser_autoconnection_task_with_mpsc,
 };
 use geyser_grpc_connector::grpcmultiplex_fastestwins::{
-    create_multiplexed_stream, FromYellowstoneExtractor,
+    FromYellowstoneExtractor,
 };
 use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
 use tokio::time::{sleep, Duration};
diff --git a/examples/stream_blocks_single.rs b/examples/stream_blocks_single.rs
index e189149..cb882fa 100644
--- a/examples/stream_blocks_single.rs
+++ b/examples/stream_blocks_single.rs
@@ -14,7 +14,6 @@ use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::SubscribeUpdate;
 use yellowstone_grpc_proto::prost::Message as _;
-use csv::Writer;
 
 #[allow(dead_code)]
 fn start_example_blockmini_consumer(
diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs
index 114a75a..9ca5c1e 100644
--- a/src/grpc_subscription_autoreconnect_streams.rs
+++ b/src/grpc_subscription_autoreconnect_streams.rs
@@ -5,9 +5,10 @@ use log::{debug, info, log, trace, warn, Level};
 use std::time::Duration;
 use tokio::task::JoinHandle;
 use tokio::time::{sleep, timeout};
-use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult};
+use yellowstone_grpc_client::GeyserGrpcClientResult;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::Status;
+use crate::yellowstone_grpc_util::{connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig};
 
 enum ConnectionState<S: Stream<Item = Result<SubscribeUpdate, Status>>> {
     NotConnected(Attempt),
@@ -45,12 +46,16 @@ pub fn create_geyser_reconnecting_stream(
                         log!(if attempt > 1 { Level::Warn } else { Level::Debug }, "Connecting attempt #{} to {}", attempt, addr);
                         async move {
 
-                            let connect_result = GeyserGrpcClient::connect_with_timeout(
-                                    addr, token, config,
-                                    connect_timeout,
-                                    request_timeout,
-                                false)
-                                .await;
+                            let connect_result = connect_with_timeout_with_buffers(
+                                addr,
+                                token,
+                                config,
+                                connect_timeout,
+                                request_timeout,
+                                GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter),
+                            )
+                            .await;
+
                             let mut client = connect_result?;
 
 
diff --git a/src/lib.rs b/src/lib.rs
index 1e13e55..27e37b7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -9,7 +9,7 @@ pub mod channel_plugger;
 pub mod grpc_subscription_autoreconnect_streams;
 pub mod grpc_subscription_autoreconnect_tasks;
 pub mod grpcmultiplex_fastestwins;
-mod yellowstone_grpc_util;
+pub mod yellowstone_grpc_util;
 mod obfuscate;
 
 // 1-based attempt counter
diff --git a/src/yellowstone_grpc_util.rs b/src/yellowstone_grpc_util.rs
index bb5d4b9..2dc7d59 100644
--- a/src/yellowstone_grpc_util.rs
+++ b/src/yellowstone_grpc_util.rs
@@ -104,7 +104,7 @@ pub async fn connect_with_timeout_with_buffers<E, T>(
     let interceptor = InterceptorXToken { x_token };
 
     let channel = endpoint.connect_lazy();
-    let mut client = GeyserGrpcClient::new(
+    let client = GeyserGrpcClient::new(
         HealthClient::with_interceptor(channel.clone(), interceptor.clone()),
         GeyserClient::with_interceptor(channel, interceptor)
             .max_decoding_message_size(GeyserGrpcClient::max_decoding_message_size()),

From e9eb3ecdc57599b9338545936eed196adf3867e3 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 10:18:04 +0200
Subject: [PATCH 04/67] relax yellowstone dependencies

---
 Cargo.lock | 10 ++++++----
 Cargo.toml |  7 +++----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1864a0e..d9368f7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4156,8 +4156,9 @@ dependencies = [
 
 [[package]]
 name = "yellowstone-grpc-client"
-version = "1.14.0+solana.1.17.28"
-source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.13.0+solana.1.17.28#aafd827df872f093535c425ded4eca423929a3d0"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f4f6d836d214cb8789002d039412da354049e9ffe983c643ec492c4d934698f"
 dependencies = [
  "bytes",
  "futures",
@@ -4170,8 +4171,9 @@ dependencies = [
 
 [[package]]
 name = "yellowstone-grpc-proto"
-version = "1.13.0+solana.1.17.28"
-source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.13.0+solana.1.17.28#aafd827df872f093535c425ded4eca423929a3d0"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c00b66d40d85c405f14b420d7674e98c70d06f6b673f36c9e0285f81b9b797d"
 dependencies = [
  "anyhow",
  "bincode",
diff --git a/Cargo.toml b/Cargo.toml
index 5a49454..6001a70 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "geyser-grpc-connector"
-version = "0.10.1+yellowstone.1.12"
+version = "0.10.4+yellowstone.1.13"
 edition = "2021"
 
 description = "Multiplexing and Reconnection on Yellowstone gRPC Geyser client streaming"
@@ -9,9 +9,8 @@ authors = ["GroovieGermanikus <groovie@mango.markets>"]
 repository = "https://github.com/blockworks-foundation/geyser-grpc-connector"
 
 [dependencies]
-yellowstone-grpc-client = { version = "1.14.0", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.13.0+solana.1.17.28" }
-yellowstone-grpc-proto = { version = "1.13.0", git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.13.0+solana.1.17.28" }
-
+yellowstone-grpc-client = "~1.14.0"
+yellowstone-grpc-proto = "~1.13.0"
 
 # required for CommitmentConfig
 solana-sdk = "1"

From 7f115fb8b5edfc8ab1134b2885c556513495b071 Mon Sep 17 00:00:00 2001
From: godmodegalactus <godmodegalactus@gmail.com>
Date: Wed, 27 Mar 2024 17:47:25 +0100
Subject: [PATCH 05/67] Adding notify channel to stop tasks

---
 Cargo.lock                                   |   2 +-
 examples/stream_blocks_autoconnect.rs        |   5 +
 examples/stream_blocks_mainnet_task.rs       |  14 +-
 src/grpc_subscription_autoreconnect_tasks.rs | 133 +++++++++++++------
 src/yellowstone_grpc_util.rs                 |  35 ++---
 5 files changed, 129 insertions(+), 60 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d9368f7..e21a5fd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1249,7 +1249,7 @@ dependencies = [
 
 [[package]]
 name = "geyser-grpc-connector"
-version = "0.10.1+yellowstone.1.12"
+version = "0.10.4+yellowstone.1.13"
 dependencies = [
  "anyhow",
  "async-stream",
diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs
index 6b3a40c..02ddda6 100644
--- a/examples/stream_blocks_autoconnect.rs
+++ b/examples/stream_blocks_autoconnect.rs
@@ -2,6 +2,8 @@ use log::info;
 use solana_sdk::clock::Slot;
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
+use std::sync::Arc;
+use tokio::sync::Notify;
 
 use geyser_grpc_connector::channel_plugger::spawn_broadcast_channel_plug;
 use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task;
@@ -87,9 +89,12 @@ pub async fn main() {
 
     info!("Write Block stream..");
 
+    let exit_notify = Arc::new(Notify::new());
+
     let (jh_geyser_task, message_channel) = create_geyser_autoconnection_task(
         green_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_and_txs(),
+        exit_notify,
     );
     let mut message_channel =
         spawn_broadcast_channel_plug(tokio::sync::broadcast::channel(8), message_channel);
diff --git a/examples/stream_blocks_mainnet_task.rs b/examples/stream_blocks_mainnet_task.rs
index 431ae2e..265bbdf 100644
--- a/examples/stream_blocks_mainnet_task.rs
+++ b/examples/stream_blocks_mainnet_task.rs
@@ -2,6 +2,8 @@ use log::{info, warn};
 use solana_sdk::clock::Slot;
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
+use std::sync::Arc;
+use tokio::sync::Notify;
 
 use base64::Engine;
 use itertools::Itertools;
@@ -20,12 +22,8 @@ use solana_sdk::transaction::TransactionError;
 use tokio::sync::mpsc::Receiver;
 use yellowstone_grpc_proto::geyser::SubscribeUpdateBlock;
 
-use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::{
-    create_geyser_autoconnection_task_with_mpsc,
-};
-use geyser_grpc_connector::grpcmultiplex_fastestwins::{
-    FromYellowstoneExtractor,
-};
+use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
 use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
 use tokio::time::{sleep, Duration};
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
@@ -122,6 +120,7 @@ pub async fn main() {
         subscribe_timeout: Duration::from_secs(5),
         receive_timeout: Duration::from_secs(5),
     };
+    let exit_notify = Arc::new(Notify::new());
 
     let green_config =
         GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
@@ -135,16 +134,19 @@ pub async fn main() {
         green_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(),
         autoconnect_tx.clone(),
+        exit_notify.clone(),
     );
     let _blue_stream_ah = create_geyser_autoconnection_task_with_mpsc(
         blue_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(),
         autoconnect_tx.clone(),
+        exit_notify.clone(),
     );
     let _toxiproxy_stream_ah = create_geyser_autoconnection_task_with_mpsc(
         toxiproxy_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(),
         autoconnect_tx.clone(),
+        exit_notify.clone(),
     );
     start_example_blockmeta_consumer(blockmeta_rx);
 
diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index 9e47887..82cbfb3 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -1,16 +1,18 @@
-use crate::{Attempt, GrpcSourceConfig, Message};
+use crate::{yellowstone_grpc_util, Attempt, GrpcSourceConfig, Message};
 use futures::{Stream, StreamExt};
 use log::{debug, error, info, log, trace, warn, Level};
+use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc::error::SendTimeoutError;
 use tokio::sync::mpsc::Receiver;
-use tokio::task::AbortHandle;
+use tokio::sync::Notify;
+use tokio::task::JoinHandle;
 use tokio::time::{sleep, timeout, Instant};
 use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError};
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::service::Interceptor;
 use yellowstone_grpc_proto::tonic::Status;
-use crate::yellowstone_grpc_util::{connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig};
+use crate::yellowstone_grpc_util::connect_with_timeout_with_buffers;
 
 enum ConnectionState<S: Stream<Item = Result<SubscribeUpdate, Status>>, F: Interceptor> {
     NotConnected(Attempt),
@@ -34,13 +36,18 @@ enum FatalErrorReason {
 pub fn create_geyser_autoconnection_task(
     grpc_source: GrpcSourceConfig,
     subscribe_filter: SubscribeRequest,
-) -> (AbortHandle, Receiver<Message>) {
+    exit_notify: Arc<Notify>,
+) -> (JoinHandle<()>, Receiver<Message>) {
     let (sender, receiver_channel) = tokio::sync::mpsc::channel::<Message>(1);
 
-    let abort_handle =
-        create_geyser_autoconnection_task_with_mpsc(grpc_source, subscribe_filter, sender);
+    let join_handle = create_geyser_autoconnection_task_with_mpsc(
+        grpc_source,
+        subscribe_filter,
+        sender,
+        exit_notify,
+    );
 
-    (abort_handle, receiver_channel)
+    (join_handle, receiver_channel)
 }
 
 /// connect to grpc source performing autoconnect if required,
@@ -50,7 +57,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
     grpc_source: GrpcSourceConfig,
     subscribe_filter: SubscribeRequest,
     mpsc_downstream: tokio::sync::mpsc::Sender<Message>,
-) -> AbortHandle {
+    exit_notify: Arc<Notify>,
+) -> JoinHandle<()> {
     // read this for argument: http://www.randomhacks.net/2019/03/08/should-rust-channels-panic-on-send/
 
     // task will be aborted when downstream receiver gets dropped
@@ -58,7 +66,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
         let mut state = ConnectionState::NotConnected(1);
         let mut messages_forwarded = 0;
 
-        loop {
+        'main_loop: loop {
             state = match state {
                 ConnectionState::NotConnected(attempt) => {
                     let addr = grpc_source.grpc_addr.clone();
@@ -77,15 +85,23 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         addr
                     );
 
-                    let connect_result = connect_with_timeout_with_buffers(
-                        addr,
-                        token,
-                        config,
-                        connect_timeout,
-                        request_timeout,
-                        GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter),
-                    )
-                    .await;
+                    let buffer_config = yellowstone_grpc_util::GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter);
+                    debug!("Using Grpc Buffer config {:?}", buffer_config);
+                    let connect_result = tokio::select! {
+                            res = connect_with_timeout_with_buffers(
+                            addr,
+                            token,
+                            config,
+                            connect_timeout,
+                            request_timeout,
+                            buffer_config,
+                        ) => {
+                            res
+                        },
+                        _ = exit_notify.notified() => {
+                            break 'main_loop;
+                        }
+                    };
 
                     match connect_result {
                         Ok(client) => ConnectionState::Connecting(attempt, client),
@@ -134,11 +150,17 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                     let subscribe_filter = subscribe_filter.clone();
                     debug!("Subscribe with filter {:?}", subscribe_filter);
 
-                    let subscribe_result_timeout = timeout(
-                        subscribe_timeout.unwrap_or(Duration::MAX),
-                        client.subscribe_once2(subscribe_filter),
-                    )
-                    .await;
+                    let subscribe_result_timeout = tokio::select! {
+                        res = timeout(
+                            subscribe_timeout.unwrap_or(Duration::MAX),
+                            client.subscribe_once2(subscribe_filter),
+                        ) => {
+                            res
+                        },
+                        _ = exit_notify.notified() => {
+                            break 'main_loop;
+                        }
+                    };
 
                     match subscribe_result_timeout {
                         Ok(subscribe_result) => {
@@ -194,7 +216,14 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         "waiting {} seconds, then reconnect to {}",
                         backoff_secs, grpc_source
                     );
-                    sleep(Duration::from_secs_f32(backoff_secs)).await;
+                    tokio::select! {
+                        _ = sleep(Duration::from_secs_f32(backoff_secs)) => {
+                            //slept
+                        },
+                        _ = exit_notify.notified() => {
+                            break 'main_loop;
+                        }
+                    };
                     ConnectionState::NotConnected(attempt)
                 }
                 ConnectionState::FatalError(_attempt, reason) => match reason {
@@ -221,18 +250,31 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         "waiting {} seconds, then reconnect to {}",
                         backoff_secs, grpc_source
                     );
-                    sleep(Duration::from_secs_f32(backoff_secs)).await;
+                    tokio::select! {
+                        _ = sleep(Duration::from_secs_f32(backoff_secs)) => {
+                            //slept
+                        },
+                        _ = exit_notify.notified() => {
+                            break 'main_loop;
+                        }
+                    };
                     ConnectionState::NotConnected(attempt)
                 }
                 ConnectionState::Ready(mut geyser_stream) => {
                     let receive_timeout = grpc_source.timeouts.as_ref().map(|t| t.receive_timeout);
                     'recv_loop: loop {
-                        match timeout(
-                            receive_timeout.unwrap_or(Duration::MAX),
-                            geyser_stream.next(),
-                        )
-                        .await
-                        {
+                        let geyser_stream_res = tokio::select! {
+                            res = timeout(
+                                receive_timeout.unwrap_or(Duration::MAX),
+                                geyser_stream.next(),
+                            ) => {
+                                res
+                            },
+                            _ = exit_notify.notified() => {
+                                break 'main_loop;
+                            }
+                        };
+                        match geyser_stream_res {
                             Ok(Some(Ok(update_message))) => {
                                 trace!("> recv update message from {}", grpc_source);
                                 // note: first send never blocks as the mpsc channel has capacity 1
@@ -242,13 +284,21 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                     Duration::from_millis(500)
                                 };
                                 let started_at = Instant::now();
-                                match mpsc_downstream
+
+                                let mpsc_downstream_result = tokio::select! {
+                                    res = mpsc_downstream
                                     .send_timeout(
                                         Message::GeyserSubscribeUpdate(Box::new(update_message)),
                                         warning_threshold,
-                                    )
-                                    .await
-                                {
+                                    ) => {
+                                        res
+                                    },
+                                    _ = exit_notify.notified() => {
+                                        break 'main_loop;
+                                    }
+                                };
+
+                                match mpsc_downstream_result {
                                     Ok(()) => {
                                         messages_forwarded += 1;
                                         if messages_forwarded == 1 {
@@ -266,7 +316,16 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                     Err(SendTimeoutError::Timeout(the_message)) => {
                                         warn!("downstream receiver did not pick up message for {}ms - keep waiting", warning_threshold.as_millis());
 
-                                        match mpsc_downstream.send(the_message).await {
+                                        let mpsc_downstream_result = tokio::select! {
+                                            res = mpsc_downstream.send(the_message)=> {
+                                                res
+                                            },
+                                            _ = exit_notify.notified() => {
+                                                break 'main_loop;
+                                            }
+                                        };
+
+                                        match mpsc_downstream_result {
                                             Ok(()) => {
                                                 messages_forwarded += 1;
                                                 trace!(
@@ -313,7 +372,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
         } // -- endless state loop
     });
 
-    jh_geyser_task.abort_handle()
+    jh_geyser_task
 }
 
 #[cfg(test)]
diff --git a/src/yellowstone_grpc_util.rs b/src/yellowstone_grpc_util.rs
index 2dc7d59..c2831a9 100644
--- a/src/yellowstone_grpc_util.rs
+++ b/src/yellowstone_grpc_util.rs
@@ -5,12 +5,11 @@ use yellowstone_grpc_proto::geyser::geyser_client::GeyserClient;
 use yellowstone_grpc_proto::geyser::SubscribeRequest;
 use yellowstone_grpc_proto::prost::bytes::Bytes;
 use yellowstone_grpc_proto::tonic;
-use yellowstone_grpc_proto::tonic::metadata::AsciiMetadataValue;
 use yellowstone_grpc_proto::tonic::metadata::errors::InvalidMetadataValue;
+use yellowstone_grpc_proto::tonic::metadata::AsciiMetadataValue;
 use yellowstone_grpc_proto::tonic::service::Interceptor;
 use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig;
 
-
 pub async fn connect_with_timeout<E, T>(
     endpoint: E,
     x_token: Option<T>,
@@ -19,15 +18,21 @@ pub async fn connect_with_timeout<E, T>(
     request_timeout: Option<Duration>,
     connect_lazy: bool,
 ) -> GeyserGrpcClientResult<GeyserGrpcClient<impl Interceptor>>
-    where
-        E: Into<Bytes>,
-        T: TryInto<AsciiMetadataValue, Error = InvalidMetadataValue>,
+where
+    E: Into<Bytes>,
+    T: TryInto<AsciiMetadataValue, Error = InvalidMetadataValue>,
 {
     GeyserGrpcClient::connect_with_timeout(
-       endpoint, x_token, tls_config, connect_timeout, request_timeout, connect_lazy).await
+        endpoint,
+        x_token,
+        tls_config,
+        connect_timeout,
+        request_timeout,
+        connect_lazy,
+    )
+    .await
 }
 
-
 // see https://github.com/hyperium/tonic/blob/v0.10.2/tonic/src/transport/channel/mod.rs
 const DEFAULT_BUFFER_SIZE: usize = 1024;
 // see https://github.com/hyperium/hyper/blob/v0.14.28/src/proto/h2/client.rs#L45
@@ -52,22 +57,19 @@ impl Default for GeyserGrpcClientBufferConfig {
 }
 
 impl GeyserGrpcClientBufferConfig {
-
     pub fn optimize_for_subscription(filter: &SubscribeRequest) -> GeyserGrpcClientBufferConfig {
         if !filter.blocks.is_empty() {
             GeyserGrpcClientBufferConfig {
-                buffer_size: Some(65536),  // 64kb (default: 1k)
-                conn_window: Some(5242880), // 5mb (=default)
+                buffer_size: Some(65536),     // 64kb (default: 1k)
+                conn_window: Some(5242880),   // 5mb (=default)
                 stream_window: Some(4194304), // 4mb (default: 2m)
             }
         } else {
             GeyserGrpcClientBufferConfig::default()
         }
     }
-
 }
 
-
 pub async fn connect_with_timeout_with_buffers<E, T>(
     endpoint: E,
     x_token: Option<T>,
@@ -76,9 +78,10 @@ pub async fn connect_with_timeout_with_buffers<E, T>(
     request_timeout: Option<Duration>,
     buffer_config: GeyserGrpcClientBufferConfig,
 ) -> GeyserGrpcClientResult<GeyserGrpcClient<impl Interceptor>>
-    where
-        E: Into<Bytes>,
-        T: TryInto<AsciiMetadataValue, Error = InvalidMetadataValue>, {
+where
+    E: Into<Bytes>,
+    T: TryInto<AsciiMetadataValue, Error = InvalidMetadataValue>,
+{
     // see https://github.com/blockworks-foundation/geyser-grpc-connector/issues/10
     let mut endpoint = tonic::transport::Endpoint::from_shared(endpoint)?
         .buffer_size(buffer_config.buffer_size)
@@ -110,4 +113,4 @@ pub async fn connect_with_timeout_with_buffers<E, T>(
             .max_decoding_message_size(GeyserGrpcClient::max_decoding_message_size()),
     );
     Ok(client)
-}
\ No newline at end of file
+}

From 280f03d4d5933a74c38bbd6ab49cc01c004a0abc Mon Sep 17 00:00:00 2001
From: godmodegalactus <godmodegalactus@gmail.com>
Date: Tue, 2 Apr 2024 14:36:22 +0200
Subject: [PATCH 06/67] Changing Notify channel to broadcast channel

---
 examples/stream_blocks_autoconnect.rs        |  5 ++---
 examples/stream_blocks_mainnet_task.rs       | 10 ++++------
 src/grpc_subscription_autoreconnect_tasks.rs | 21 ++++++++++----------
 3 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/examples/stream_blocks_autoconnect.rs b/examples/stream_blocks_autoconnect.rs
index 02ddda6..09214be 100644
--- a/examples/stream_blocks_autoconnect.rs
+++ b/examples/stream_blocks_autoconnect.rs
@@ -2,8 +2,7 @@ use log::info;
 use solana_sdk::clock::Slot;
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
-use std::sync::Arc;
-use tokio::sync::Notify;
+use tokio::sync::broadcast;
 
 use geyser_grpc_connector::channel_plugger::spawn_broadcast_channel_plug;
 use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task;
@@ -89,7 +88,7 @@ pub async fn main() {
 
     info!("Write Block stream..");
 
-    let exit_notify = Arc::new(Notify::new());
+    let (_, exit_notify) = broadcast::channel(1);
 
     let (jh_geyser_task, message_channel) = create_geyser_autoconnection_task(
         green_config.clone(),
diff --git a/examples/stream_blocks_mainnet_task.rs b/examples/stream_blocks_mainnet_task.rs
index 265bbdf..380a937 100644
--- a/examples/stream_blocks_mainnet_task.rs
+++ b/examples/stream_blocks_mainnet_task.rs
@@ -2,8 +2,6 @@ use log::{info, warn};
 use solana_sdk::clock::Slot;
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
-use std::sync::Arc;
-use tokio::sync::Notify;
 
 use base64::Engine;
 use itertools::Itertools;
@@ -120,7 +118,7 @@ pub async fn main() {
         subscribe_timeout: Duration::from_secs(5),
         receive_timeout: Duration::from_secs(5),
     };
-    let exit_notify = Arc::new(Notify::new());
+    let (_, exit_notify) = tokio::sync::broadcast::channel(1);
 
     let green_config =
         GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
@@ -134,19 +132,19 @@ pub async fn main() {
         green_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(),
         autoconnect_tx.clone(),
-        exit_notify.clone(),
+        exit_notify.resubscribe(),
     );
     let _blue_stream_ah = create_geyser_autoconnection_task_with_mpsc(
         blue_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(),
         autoconnect_tx.clone(),
-        exit_notify.clone(),
+        exit_notify.resubscribe(),
     );
     let _toxiproxy_stream_ah = create_geyser_autoconnection_task_with_mpsc(
         toxiproxy_config.clone(),
         GeyserFilter(CommitmentConfig::confirmed()).blocks_meta(),
         autoconnect_tx.clone(),
-        exit_notify.clone(),
+        exit_notify,
     );
     start_example_blockmeta_consumer(blockmeta_rx);
 
diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index 82cbfb3..197afaf 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -1,11 +1,10 @@
 use crate::{yellowstone_grpc_util, Attempt, GrpcSourceConfig, Message};
 use futures::{Stream, StreamExt};
 use log::{debug, error, info, log, trace, warn, Level};
-use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc::error::SendTimeoutError;
 use tokio::sync::mpsc::Receiver;
-use tokio::sync::Notify;
+use tokio::sync::broadcast;
 use tokio::task::JoinHandle;
 use tokio::time::{sleep, timeout, Instant};
 use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError};
@@ -36,7 +35,7 @@ enum FatalErrorReason {
 pub fn create_geyser_autoconnection_task(
     grpc_source: GrpcSourceConfig,
     subscribe_filter: SubscribeRequest,
-    exit_notify: Arc<Notify>,
+    exit_notify: broadcast::Receiver<()>,
 ) -> (JoinHandle<()>, Receiver<Message>) {
     let (sender, receiver_channel) = tokio::sync::mpsc::channel::<Message>(1);
 
@@ -57,7 +56,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
     grpc_source: GrpcSourceConfig,
     subscribe_filter: SubscribeRequest,
     mpsc_downstream: tokio::sync::mpsc::Sender<Message>,
-    exit_notify: Arc<Notify>,
+    mut exit_notify: broadcast::Receiver<()>,
 ) -> JoinHandle<()> {
     // read this for argument: http://www.randomhacks.net/2019/03/08/should-rust-channels-panic-on-send/
 
@@ -98,7 +97,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         ) => {
                             res
                         },
-                        _ = exit_notify.notified() => {
+                        _ = exit_notify.recv() => {
                             break 'main_loop;
                         }
                     };
@@ -157,7 +156,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         ) => {
                             res
                         },
-                        _ = exit_notify.notified() => {
+                        _ = exit_notify.recv() => {
                             break 'main_loop;
                         }
                     };
@@ -220,7 +219,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         _ = sleep(Duration::from_secs_f32(backoff_secs)) => {
                             //slept
                         },
-                        _ = exit_notify.notified() => {
+                        _ = exit_notify.recv() => {
                             break 'main_loop;
                         }
                     };
@@ -254,7 +253,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         _ = sleep(Duration::from_secs_f32(backoff_secs)) => {
                             //slept
                         },
-                        _ = exit_notify.notified() => {
+                        _ = exit_notify.recv() => {
                             break 'main_loop;
                         }
                     };
@@ -270,7 +269,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                             ) => {
                                 res
                             },
-                            _ = exit_notify.notified() => {
+                            _ = exit_notify.recv() => {
                                 break 'main_loop;
                             }
                         };
@@ -293,7 +292,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                     ) => {
                                         res
                                     },
-                                    _ = exit_notify.notified() => {
+                                    _ = exit_notify.recv() => {
                                         break 'main_loop;
                                     }
                                 };
@@ -320,7 +319,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                             res = mpsc_downstream.send(the_message)=> {
                                                 res
                                             },
-                                            _ = exit_notify.notified() => {
+                                            _ = exit_notify.recv() => {
                                                 break 'main_loop;
                                             }
                                         };

From e000a9674749e9408aedc4f002c9abc128b53871 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 10:45:50 +0200
Subject: [PATCH 07/67] bump version to 0.10.5

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 6001a70..15ed67a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "geyser-grpc-connector"
-version = "0.10.4+yellowstone.1.13"
+version = "0.10.5+yellowstone.1.13"
 edition = "2021"
 
 description = "Multiplexing and Reconnection on Yellowstone gRPC Geyser client streaming"

From aad4baf0ea7769ae40913f61fc3afbadf57b3a53 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 12:17:12 +0200
Subject: [PATCH 08/67] cargo

---
 Cargo.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index e21a5fd..399fc2f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1249,7 +1249,7 @@ dependencies = [
 
 [[package]]
 name = "geyser-grpc-connector"
-version = "0.10.4+yellowstone.1.13"
+version = "0.10.5+yellowstone.1.13"
 dependencies = [
  "anyhow",
  "async-stream",

From fef2e70520a8106430bf7236f64e14b7534eb4e7 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 11:51:03 +0200
Subject: [PATCH 09/67] cleanup

---
 src/grpc_subscription_autoreconnect_tasks.rs | 170 ++++++++++---------
 1 file changed, 93 insertions(+), 77 deletions(-)

diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index 197afaf..d07155a 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -1,3 +1,4 @@
+use std::future::Future;
 use crate::{yellowstone_grpc_util, Attempt, GrpcSourceConfig, Message};
 use futures::{Stream, StreamExt};
 use log::{debug, error, info, log, trace, warn, Level};
@@ -23,6 +24,8 @@ enum ConnectionState<S: Stream<Item = Result<SubscribeUpdate, Status>>, F: Inter
     // non-recoverable error
     FatalError(Attempt, FatalErrorReason),
     WaitReconnect(Attempt),
+    // exit signal received
+    GracefulShutdown,
 }
 
 enum FatalErrorReason {
@@ -86,23 +89,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
 
                     let buffer_config = yellowstone_grpc_util::GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter);
                     debug!("Using Grpc Buffer config {:?}", buffer_config);
-                    let connect_result = tokio::select! {
-                            res = connect_with_timeout_with_buffers(
-                            addr,
-                            token,
-                            config,
-                            connect_timeout,
-                            request_timeout,
-                            buffer_config,
-                        ) => {
-                            res
-                        },
-                        _ = exit_notify.recv() => {
-                            break 'main_loop;
-                        }
-                    };
 
-                    match connect_result {
+                    let connection_handler = |connect_result| match connect_result {
                         Ok(client) => ConnectionState::Connecting(attempt, client),
                         Err(GeyserGrpcClientError::InvalidUri(_)) => ConnectionState::FatalError(
                             attempt + 1,
@@ -141,7 +129,22 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                             );
                             ConnectionState::RecoverableConnectionError(attempt + 1)
                         }
+                    };
+
+                    let fut_connector = connect_with_timeout_with_buffers(
+                        addr,
+                        token,
+                        config,
+                        connect_timeout,
+                        request_timeout,
+                        buffer_config,
+                    );
+
+                    match await_or_exit(fut_connector, exit_notify.recv()).await {
+                        Some(connection_result) => connection_handler(connection_result),
+                        None => ConnectionState::GracefulShutdown
                     }
+
                 }
                 ConnectionState::Connecting(attempt, mut client) => {
                     let subscribe_timeout =
@@ -149,19 +152,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                     let subscribe_filter = subscribe_filter.clone();
                     debug!("Subscribe with filter {:?}", subscribe_filter);
 
-                    let subscribe_result_timeout = tokio::select! {
-                        res = timeout(
-                            subscribe_timeout.unwrap_or(Duration::MAX),
-                            client.subscribe_once2(subscribe_filter),
-                        ) => {
-                            res
-                        },
-                        _ = exit_notify.recv() => {
-                            break 'main_loop;
-                        }
-                    };
-
-                    match subscribe_result_timeout {
+                    let subscribe_handler = |subscribe_result_timeout| match subscribe_result_timeout {
                         Ok(subscribe_result) => {
                             match subscribe_result {
                                 Ok(geyser_stream) => {
@@ -207,7 +198,18 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                             );
                             ConnectionState::RecoverableConnectionError(attempt + 1)
                         }
+                    };
+
+                    let fut_subscribe = timeout(
+                        subscribe_timeout.unwrap_or(Duration::MAX),
+                        client.subscribe_once2(subscribe_filter),
+                    );
+
+                    match await_or_exit(fut_subscribe, exit_notify.recv()).await {
+                        Some(subscribe_result_timeout) => subscribe_handler(subscribe_result_timeout),
+                        None => ConnectionState::GracefulShutdown
                     }
+
                 }
                 ConnectionState::RecoverableConnectionError(attempt) => {
                     let backoff_secs = 1.5_f32.powi(attempt as i32).min(15.0);
@@ -215,15 +217,13 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         "waiting {} seconds, then reconnect to {}",
                         backoff_secs, grpc_source
                     );
-                    tokio::select! {
-                        _ = sleep(Duration::from_secs_f32(backoff_secs)) => {
-                            //slept
-                        },
-                        _ = exit_notify.recv() => {
-                            break 'main_loop;
-                        }
-                    };
-                    ConnectionState::NotConnected(attempt)
+
+                    let fut_sleep = sleep(Duration::from_secs_f32(backoff_secs));
+
+                    match await_or_exit(fut_sleep, exit_notify.recv()).await {
+                        Some(()) => ConnectionState::NotConnected(attempt),
+                        None => ConnectionState::GracefulShutdown
+                    }
                 }
                 ConnectionState::FatalError(_attempt, reason) => match reason {
                     FatalErrorReason::DownstreamChannelClosed => {
@@ -249,30 +249,27 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         "waiting {} seconds, then reconnect to {}",
                         backoff_secs, grpc_source
                     );
-                    tokio::select! {
-                        _ = sleep(Duration::from_secs_f32(backoff_secs)) => {
-                            //slept
-                        },
-                        _ = exit_notify.recv() => {
-                            break 'main_loop;
-                        }
-                    };
-                    ConnectionState::NotConnected(attempt)
+
+                    let fut_sleep = sleep(Duration::from_secs_f32(backoff_secs));
+
+                    match await_or_exit(fut_sleep, exit_notify.recv()).await {
+                        Some(()) => ConnectionState::NotConnected(attempt),
+                        None => ConnectionState::GracefulShutdown
+                    }
                 }
                 ConnectionState::Ready(mut geyser_stream) => {
                     let receive_timeout = grpc_source.timeouts.as_ref().map(|t| t.receive_timeout);
                     'recv_loop: loop {
-                        let geyser_stream_res = tokio::select! {
-                            res = timeout(
-                                receive_timeout.unwrap_or(Duration::MAX),
-                                geyser_stream.next(),
-                            ) => {
-                                res
-                            },
-                            _ = exit_notify.recv() => {
-                                break 'main_loop;
-                            }
+
+                        let fut_stream = timeout(
+                            receive_timeout.unwrap_or(Duration::MAX),
+                            geyser_stream.next(),
+                        );
+
+                        let Some(geyser_stream_res) = await_or_exit(fut_stream, exit_notify.recv()).await else {
+                            break 'recv_loop ConnectionState::GracefulShutdown;
                         };
+
                         match geyser_stream_res {
                             Ok(Some(Ok(update_message))) => {
                                 trace!("> recv update message from {}", grpc_source);
@@ -284,17 +281,15 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                 };
                                 let started_at = Instant::now();
 
-                                let mpsc_downstream_result = tokio::select! {
-                                    res = mpsc_downstream
-                                    .send_timeout(
-                                        Message::GeyserSubscribeUpdate(Box::new(update_message)),
-                                        warning_threshold,
-                                    ) => {
-                                        res
-                                    },
-                                    _ = exit_notify.recv() => {
-                                        break 'main_loop;
-                                    }
+                                let fut_send = mpsc_downstream.send_timeout(
+                                    Message::GeyserSubscribeUpdate(Box::new(update_message)),
+                                    warning_threshold,
+                                );
+
+                                let Some(mpsc_downstream_result) = await_or_exit(
+                                    fut_send,
+                                    exit_notify.recv()).await else {
+                                    break 'recv_loop ConnectionState::GracefulShutdown;
                                 };
 
                                 match mpsc_downstream_result {
@@ -315,13 +310,12 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                     Err(SendTimeoutError::Timeout(the_message)) => {
                                         warn!("downstream receiver did not pick up message for {}ms - keep waiting", warning_threshold.as_millis());
 
-                                        let mpsc_downstream_result = tokio::select! {
-                                            res = mpsc_downstream.send(the_message)=> {
-                                                res
-                                            },
-                                            _ = exit_notify.recv() => {
-                                                break 'main_loop;
-                                            }
+                                        let fut_send = mpsc_downstream.send(the_message);
+
+                                        let Some(mpsc_downstream_result) = await_or_exit(
+                                            fut_send,
+                                            exit_notify.recv()).await else {
+                                            break 'recv_loop ConnectionState::GracefulShutdown;
                                         };
 
                                         match mpsc_downstream_result {
@@ -364,16 +358,38 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                 warn!("timeout on {} - retrying", grpc_source);
                                 break 'recv_loop ConnectionState::WaitReconnect(1);
                             }
-                        } // -- END match
+                        }; // -- END match
+
                     } // -- END receive loop
                 }
+                ConnectionState::GracefulShutdown => {
+                    debug!("shutting down {} gracefully on exit signal", grpc_source);
+                    break 'main_loop;
+                }
             } // -- END match
-        } // -- endless state loop
+        } // -- state loop; break ONLY on graceful shutdown
     });
 
     jh_geyser_task
 }
 
+
+async fn await_or_exit<F, E>(future: F, exit_notify: E) -> Option<F::Output>
+    where
+        F: Future,
+        E: Future,
+{
+    tokio::select! {
+        res = future => {
+            Some(res)
+        },
+        _ = exit_notify => {
+            None
+        }
+    }
+}
+
+
 #[cfg(test)]
 mod tests {
     use super::*;

From dd18a2074dd4145c84af20fef8398fd5c8044959 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 12:29:00 +0200
Subject: [PATCH 10/67] use dedicated AST

---
 src/grpc_subscription_autoreconnect_tasks.rs | 33 +++++++++++---------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index d07155a..c5727d4 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -141,8 +141,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                     );
 
                     match await_or_exit(fut_connector, exit_notify.recv()).await {
-                        Some(connection_result) => connection_handler(connection_result),
-                        None => ConnectionState::GracefulShutdown
+                        MaybeExit::Continue(connection_result) => connection_handler(connection_result),
+                        MaybeExit::Exit => ConnectionState::GracefulShutdown
                     }
 
                 }
@@ -206,8 +206,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                     );
 
                     match await_or_exit(fut_subscribe, exit_notify.recv()).await {
-                        Some(subscribe_result_timeout) => subscribe_handler(subscribe_result_timeout),
-                        None => ConnectionState::GracefulShutdown
+                        MaybeExit::Continue(subscribe_result_timeout) => subscribe_handler(subscribe_result_timeout),
+                        MaybeExit::Exit => ConnectionState::GracefulShutdown
                     }
 
                 }
@@ -221,8 +221,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                     let fut_sleep = sleep(Duration::from_secs_f32(backoff_secs));
 
                     match await_or_exit(fut_sleep, exit_notify.recv()).await {
-                        Some(()) => ConnectionState::NotConnected(attempt),
-                        None => ConnectionState::GracefulShutdown
+                        MaybeExit::Continue(()) => ConnectionState::NotConnected(attempt),
+                        MaybeExit::Exit => ConnectionState::GracefulShutdown
                     }
                 }
                 ConnectionState::FatalError(_attempt, reason) => match reason {
@@ -253,8 +253,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                     let fut_sleep = sleep(Duration::from_secs_f32(backoff_secs));
 
                     match await_or_exit(fut_sleep, exit_notify.recv()).await {
-                        Some(()) => ConnectionState::NotConnected(attempt),
-                        None => ConnectionState::GracefulShutdown
+                        MaybeExit::Continue(()) => ConnectionState::NotConnected(attempt),
+                        MaybeExit::Exit => ConnectionState::GracefulShutdown
                     }
                 }
                 ConnectionState::Ready(mut geyser_stream) => {
@@ -266,7 +266,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                             geyser_stream.next(),
                         );
 
-                        let Some(geyser_stream_res) = await_or_exit(fut_stream, exit_notify.recv()).await else {
+                        let MaybeExit::Continue(geyser_stream_res) = await_or_exit(fut_stream, exit_notify.recv()).await else {
                             break 'recv_loop ConnectionState::GracefulShutdown;
                         };
 
@@ -286,7 +286,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                                     warning_threshold,
                                 );
 
-                                let Some(mpsc_downstream_result) = await_or_exit(
+                                let MaybeExit::Continue(mpsc_downstream_result) = await_or_exit(
                                     fut_send,
                                     exit_notify.recv()).await else {
                                     break 'recv_loop ConnectionState::GracefulShutdown;
@@ -312,7 +312,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
 
                                         let fut_send = mpsc_downstream.send(the_message);
 
-                                        let Some(mpsc_downstream_result) = await_or_exit(
+                                        let MaybeExit::Continue(mpsc_downstream_result) = await_or_exit(
                                             fut_send,
                                             exit_notify.recv()).await else {
                                             break 'recv_loop ConnectionState::GracefulShutdown;
@@ -374,17 +374,22 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
 }
 
 
-async fn await_or_exit<F, E>(future: F, exit_notify: E) -> Option<F::Output>
+enum MaybeExit<T> {
+    Continue(T),
+    Exit,
+}
+
+async fn await_or_exit<F, E>(future: F, exit_notify: E) -> MaybeExit<F::Output>
     where
         F: Future,
         E: Future,
 {
     tokio::select! {
         res = future => {
-            Some(res)
+            MaybeExit::Continue(res)
         },
         _ = exit_notify => {
-            None
+            MaybeExit::Exit
         }
     }
 }

From 85d04ac1117e2fdb84d7545f1c5b4cdf1295ac1d Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 13:37:54 +0200
Subject: [PATCH 11/67] export GeyserGrpcClient, GeyserGrpcClientResult,
 GeyserGrpcClientError

---
 src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib.rs b/src/lib.rs
index 27e37b7..1ec5715 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,6 +5,8 @@ use std::time::Duration;
 use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig;
 
+pub use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult, GeyserGrpcClientError};
+
 pub mod channel_plugger;
 pub mod grpc_subscription_autoreconnect_streams;
 pub mod grpc_subscription_autoreconnect_tasks;

From 20f29c1e21e1682f124c0a3386cd4a85b1854f01 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 16 Apr 2024 13:38:18 +0200
Subject: [PATCH 12/67] bump version to 0.10.6

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 15ed67a..95ca69d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "geyser-grpc-connector"
-version = "0.10.5+yellowstone.1.13"
+version = "0.10.6+yellowstone.1.13"
 edition = "2021"
 
 description = "Multiplexing and Reconnection on Yellowstone gRPC Geyser client streaming"

From dd78804e5aa7362ce6a83d4d183c5393d6f1b6d2 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 09:08:09 +0200
Subject: [PATCH 13/67] clarify await_or_exit behavior on channel close

---
 src/grpc_subscription_autoreconnect_tasks.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index c5727d4..f15741c 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -6,6 +6,7 @@ use std::time::Duration;
 use tokio::sync::mpsc::error::SendTimeoutError;
 use tokio::sync::mpsc::Receiver;
 use tokio::sync::broadcast;
+use tokio::sync::broadcast::error::RecvError;
 use tokio::task::JoinHandle;
 use tokio::time::{sleep, timeout, Instant};
 use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError};
@@ -379,16 +380,27 @@ enum MaybeExit<T> {
     Exit,
 }
 
-async fn await_or_exit<F, E>(future: F, exit_notify: E) -> MaybeExit<F::Output>
+async fn await_or_exit<F, E, T>(future: F, exit_notify: E) -> MaybeExit<F::Output>
     where
         F: Future,
-        E: Future,
+        E: Future<Output = Result<T, RecvError>>,
 {
     tokio::select! {
         res = future => {
             MaybeExit::Continue(res)
         },
-        _ = exit_notify => {
+        res = exit_notify => {
+            match res {
+                Ok(_) => {
+                    debug!("exit on signal");
+                }
+                 Err(RecvError::Lagged(_)) => {
+                    warn!("exit on signal (lag)");
+                }
+                Err(RecvError::Closed) => {
+                    warn!("exit on signal (channel close)");
+                }
+            }
             MaybeExit::Exit
         }
     }

From 1d1a01c8aed90aedbac72f40940c22e00ad30d4c Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 09:08:09 +0200
Subject: [PATCH 14/67] clarify await_or_exit behavior on channel close

---
 src/grpc_subscription_autoreconnect_tasks.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index c5727d4..f15741c 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -6,6 +6,7 @@ use std::time::Duration;
 use tokio::sync::mpsc::error::SendTimeoutError;
 use tokio::sync::mpsc::Receiver;
 use tokio::sync::broadcast;
+use tokio::sync::broadcast::error::RecvError;
 use tokio::task::JoinHandle;
 use tokio::time::{sleep, timeout, Instant};
 use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError};
@@ -379,16 +380,27 @@ enum MaybeExit<T> {
     Exit,
 }
 
-async fn await_or_exit<F, E>(future: F, exit_notify: E) -> MaybeExit<F::Output>
+async fn await_or_exit<F, E, T>(future: F, exit_notify: E) -> MaybeExit<F::Output>
     where
         F: Future,
-        E: Future,
+        E: Future<Output = Result<T, RecvError>>,
 {
     tokio::select! {
         res = future => {
             MaybeExit::Continue(res)
         },
-        _ = exit_notify => {
+        res = exit_notify => {
+            match res {
+                Ok(_) => {
+                    debug!("exit on signal");
+                }
+                 Err(RecvError::Lagged(_)) => {
+                    warn!("exit on signal (lag)");
+                }
+                Err(RecvError::Closed) => {
+                    warn!("exit on signal (channel close)");
+                }
+            }
             MaybeExit::Exit
         }
     }

From 929b20edee64ab24e5b6b7ed619151d7afa12a3e Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 09:09:27 +0200
Subject: [PATCH 15/67] setup simple account subscription

---
 examples/bench_geyser_grpc_accounts.rs | 141 +++++++++++++++++++++++++
 1 file changed, 141 insertions(+)
 create mode 100644 examples/bench_geyser_grpc_accounts.rs

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
new file mode 100644
index 0000000..7934abf
--- /dev/null
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -0,0 +1,141 @@
+use futures::{Stream, StreamExt};
+use log::info;
+use solana_sdk::clock::Slot;
+use solana_sdk::commitment_config::CommitmentConfig;
+use solana_sdk::pubkey::Pubkey;
+use std::env;
+use std::pin::pin;
+use tokio::sync::mpsc::Receiver;
+
+use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
+use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
+use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
+use tokio::time::{sleep, Duration};
+use tracing::warn;
+use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
+use yellowstone_grpc_proto::geyser::SubscribeUpdate;
+use yellowstone_grpc_proto::prost::Message as _;
+
+fn start_example_account_consumer(mut multiplex_channel: Receiver<Message>) {
+    tokio::spawn(async move {
+        loop {
+            match multiplex_channel.recv().await {
+                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
+                    Some(UpdateOneof::Account(update)) => {
+                        let account_info = update.account.unwrap();
+                        let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
+                        info!(
+                            "got account update (green)!!! {} - {:?} - {} bytes",
+                            update.slot,
+                            account_pk,
+                            account_info.data.len()
+                        );
+                        let bytes: [u8; 32] = account_pk.to_bytes();
+                    }
+                    None => {}
+                    _ => {}
+                },
+                None => {
+                    log::warn!("multiplexer channel closed - aborting");
+                    return;
+                }
+                Some(Message::Connecting(_)) => {}
+            }
+        }
+    });
+}
+
+#[allow(dead_code)]
+fn start_example_blockmini_consumer(
+    multiplex_stream: impl Stream<Item = BlockMini> + Send + 'static,
+) {
+    tokio::spawn(async move {
+        let mut blockmeta_stream = pin!(multiplex_stream);
+        while let Some(mini) = blockmeta_stream.next().await {
+            info!(
+                "emitted block mini #{}@{} with {} bytes from multiplexer",
+                mini.slot, mini.commitment_config.commitment, mini.blocksize
+            );
+        }
+    });
+}
+
+pub struct BlockMini {
+    pub blocksize: usize,
+    pub slot: Slot,
+    pub commitment_config: CommitmentConfig,
+}
+
+struct BlockMiniExtractor(CommitmentConfig);
+
+impl FromYellowstoneExtractor for BlockMiniExtractor {
+    type Target = BlockMini;
+    fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(Slot, Self::Target)> {
+        match update.update_oneof {
+            Some(UpdateOneof::Block(update_block_message)) => {
+                let blocksize = update_block_message.encoded_len();
+                let slot = update_block_message.slot;
+                let mini = BlockMini {
+                    blocksize,
+                    slot,
+                    commitment_config: self.0,
+                };
+                Some((slot, mini))
+            }
+            Some(UpdateOneof::BlockMeta(update_blockmeta_message)) => {
+                let blocksize = update_blockmeta_message.encoded_len();
+                let slot = update_blockmeta_message.slot;
+                let mini = BlockMini {
+                    blocksize,
+                    slot,
+                    commitment_config: self.0,
+                };
+                Some((slot, mini))
+            }
+            _ => None,
+        }
+    }
+}
+
+#[tokio::main]
+pub async fn main() {
+    // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
+    tracing_subscriber::fmt::init();
+    // console_subscriber::init();
+
+    let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green");
+    let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok();
+
+    info!(
+        "Using grpc source on {} ({})",
+        grpc_addr_green,
+        grpc_x_token_green.is_some()
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(25),
+        request_timeout: Duration::from_secs(25),
+        subscribe_timeout: Duration::from_secs(25),
+        receive_timeout: Duration::from_secs(25),
+    };
+
+    let config = GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
+
+    info!("Write Block stream..");
+
+    let (autoconnect_tx, accounts_rx) = tokio::sync::mpsc::channel(10);
+    let (_exit, exit_notify) = tokio::sync::broadcast::channel(1);
+
+    let _accounts_task = create_geyser_autoconnection_task_with_mpsc(
+        config.clone(),
+        GeyserFilter(CommitmentConfig::processed()).accounts(),
+        autoconnect_tx.clone(),
+        exit_notify.resubscribe(),
+    );
+
+    start_example_account_consumer(accounts_rx);
+
+    // "infinite" sleep
+    sleep(Duration::from_secs(1800)).await;
+}

From f280f903d7e2792da1dc96925c1d4e3fa498b8cb Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 09:12:53 +0200
Subject: [PATCH 16/67] cleanup

---
 examples/bench_geyser_grpc_accounts.rs | 113 +++++++------------------
 1 file changed, 31 insertions(+), 82 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 7934abf..cc92852 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -17,87 +17,6 @@ use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::SubscribeUpdate;
 use yellowstone_grpc_proto::prost::Message as _;
 
-fn start_example_account_consumer(mut multiplex_channel: Receiver<Message>) {
-    tokio::spawn(async move {
-        loop {
-            match multiplex_channel.recv().await {
-                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
-                    Some(UpdateOneof::Account(update)) => {
-                        let account_info = update.account.unwrap();
-                        let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
-                        info!(
-                            "got account update (green)!!! {} - {:?} - {} bytes",
-                            update.slot,
-                            account_pk,
-                            account_info.data.len()
-                        );
-                        let bytes: [u8; 32] = account_pk.to_bytes();
-                    }
-                    None => {}
-                    _ => {}
-                },
-                None => {
-                    log::warn!("multiplexer channel closed - aborting");
-                    return;
-                }
-                Some(Message::Connecting(_)) => {}
-            }
-        }
-    });
-}
-
-#[allow(dead_code)]
-fn start_example_blockmini_consumer(
-    multiplex_stream: impl Stream<Item = BlockMini> + Send + 'static,
-) {
-    tokio::spawn(async move {
-        let mut blockmeta_stream = pin!(multiplex_stream);
-        while let Some(mini) = blockmeta_stream.next().await {
-            info!(
-                "emitted block mini #{}@{} with {} bytes from multiplexer",
-                mini.slot, mini.commitment_config.commitment, mini.blocksize
-            );
-        }
-    });
-}
-
-pub struct BlockMini {
-    pub blocksize: usize,
-    pub slot: Slot,
-    pub commitment_config: CommitmentConfig,
-}
-
-struct BlockMiniExtractor(CommitmentConfig);
-
-impl FromYellowstoneExtractor for BlockMiniExtractor {
-    type Target = BlockMini;
-    fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(Slot, Self::Target)> {
-        match update.update_oneof {
-            Some(UpdateOneof::Block(update_block_message)) => {
-                let blocksize = update_block_message.encoded_len();
-                let slot = update_block_message.slot;
-                let mini = BlockMini {
-                    blocksize,
-                    slot,
-                    commitment_config: self.0,
-                };
-                Some((slot, mini))
-            }
-            Some(UpdateOneof::BlockMeta(update_blockmeta_message)) => {
-                let blocksize = update_blockmeta_message.encoded_len();
-                let slot = update_blockmeta_message.slot;
-                let mini = BlockMini {
-                    blocksize,
-                    slot,
-                    commitment_config: self.0,
-                };
-                Some((slot, mini))
-            }
-            _ => None,
-        }
-    }
-}
-
 #[tokio::main]
 pub async fn main() {
     // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
@@ -134,8 +53,38 @@ pub async fn main() {
         exit_notify.resubscribe(),
     );
 
-    start_example_account_consumer(accounts_rx);
+    start_tracking_account_consumer(accounts_rx);
 
     // "infinite" sleep
     sleep(Duration::from_secs(1800)).await;
 }
+
+
+fn start_tracking_account_consumer(mut multiplex_channel: Receiver<Message>) {
+    tokio::spawn(async move {
+        loop {
+            match multiplex_channel.recv().await {
+                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
+                    Some(UpdateOneof::Account(update)) => {
+                        let account_info = update.account.unwrap();
+                        let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
+                        info!(
+                            "got account update (green)!!! {} - {:?} - {} bytes",
+                            update.slot,
+                            account_pk,
+                            account_info.data.len()
+                        );
+                        let bytes: [u8; 32] = account_pk.to_bytes();
+                    }
+                    None => {}
+                    _ => {}
+                },
+                None => {
+                    log::warn!("multiplexer channel closed - aborting");
+                    return;
+                }
+                Some(Message::Connecting(_)) => {}
+            }
+        }
+    });
+}

From e09e5e2d36cd15b05d553a8d22c6889bfcfd7a25 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 09:58:39 +0200
Subject: [PATCH 17/67] collectors

---
 src/grpc_subscription_autoreconnect_streams.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/grpc_subscription_autoreconnect_streams.rs b/src/grpc_subscription_autoreconnect_streams.rs
index 9ca5c1e..3096270 100644
--- a/src/grpc_subscription_autoreconnect_streams.rs
+++ b/src/grpc_subscription_autoreconnect_streams.rs
@@ -1,3 +1,6 @@
+use crate::yellowstone_grpc_util::{
+    connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig,
+};
 use crate::{Attempt, GrpcSourceConfig, Message};
 use async_stream::stream;
 use futures::{Stream, StreamExt};
@@ -8,7 +11,6 @@ use tokio::time::{sleep, timeout};
 use yellowstone_grpc_client::GeyserGrpcClientResult;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::Status;
-use crate::yellowstone_grpc_util::{connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig};
 
 enum ConnectionState<S: Stream<Item = Result<SubscribeUpdate, Status>>> {
     NotConnected(Attempt),

From 13935272597c4b8816f6df04247c08ba5833bb8f Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 14:49:34 +0200
Subject: [PATCH 18/67] calculate delta

---
 examples/bench_geyser_grpc_accounts.rs | 90 ++++++++++++++++++++++----
 src/lib.rs                             |  3 +-
 2 files changed, 80 insertions(+), 13 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index cc92852..09dde99 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -1,3 +1,4 @@
+use std::collections::{HashMap, VecDeque};
 use futures::{Stream, StreamExt};
 use log::info;
 use solana_sdk::clock::Slot;
@@ -5,12 +6,13 @@ use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::pubkey::Pubkey;
 use std::env;
 use std::pin::pin;
+use itertools::Itertools;
 use tokio::sync::mpsc::Receiver;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
 use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
 use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
-use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
+use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, histogram_percentiles, Message};
 use tokio::time::{sleep, Duration};
 use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
@@ -43,7 +45,7 @@ pub async fn main() {
 
     info!("Write Block stream..");
 
-    let (autoconnect_tx, accounts_rx) = tokio::sync::mpsc::channel(10);
+    let (autoconnect_tx, geyser_messages_rx) = tokio::sync::mpsc::channel(10);
     let (_exit, exit_notify) = tokio::sync::broadcast::channel(1);
 
     let _accounts_task = create_geyser_autoconnection_task_with_mpsc(
@@ -53,28 +55,92 @@ pub async fn main() {
         exit_notify.resubscribe(),
     );
 
-    start_tracking_account_consumer(accounts_rx);
+    let _slots_task = create_geyser_autoconnection_task_with_mpsc(
+        config.clone(),
+        GeyserFilter(CommitmentConfig::processed()).slots(),
+        autoconnect_tx.clone(),
+        exit_notify.resubscribe(),
+    );
+
+    start_tracking_account_consumer(geyser_messages_rx);
 
     // "infinite" sleep
     sleep(Duration::from_secs(1800)).await;
 }
 
 
-fn start_tracking_account_consumer(mut multiplex_channel: Receiver<Message>) {
+fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
+    const RECENT_SLOTS_LIMIT: usize = 30;
+
     tokio::spawn(async move {
+
+        let mut bytes_per_slot = HashMap::<Slot, usize>::new();
+        let mut updates_per_slot = HashMap::<Slot, usize>::new();
+        let mut count_updates_per_slot_account = HashMap::<(Slot, Pubkey), usize>::new();
+        // slot written by account update
+        let mut current_slot: Slot = 0;
+        // slot from slot stream
+        let mut actual_slot: Slot = 0;
+
+        let mut recent_slot_deltas: VecDeque<i64> = VecDeque::with_capacity(RECENT_SLOTS_LIMIT);
+
         loop {
-            match multiplex_channel.recv().await {
+            match geyser_messages_rx.recv().await {
                 Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
                     Some(UpdateOneof::Account(update)) => {
                         let account_info = update.account.unwrap();
                         let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
-                        info!(
-                            "got account update (green)!!! {} - {:?} - {} bytes",
-                            update.slot,
-                            account_pk,
-                            account_info.data.len()
-                        );
-                        let bytes: [u8; 32] = account_pk.to_bytes();
+                        let slot = update.slot;
+
+                        if actual_slot != slot {
+                            if actual_slot != 0 {
+                                recent_slot_deltas.push_back((actual_slot as i64) - (slot as i64));
+                                if recent_slot_deltas.len() > RECENT_SLOTS_LIMIT {
+                                    recent_slot_deltas.pop_front();
+                                }
+                            }
+                        }
+
+                        bytes_per_slot.entry(slot)
+                            .and_modify(|entry| *entry += account_info.data.len())
+                            .or_insert(account_info.data.len());
+                        updates_per_slot.entry(slot)
+                            .and_modify(|entry| *entry += 1)
+                            .or_insert(1);
+                        count_updates_per_slot_account.entry((slot, account_pk))
+                            .and_modify(|entry| *entry += 1)
+                            .or_insert(1);
+
+                        if current_slot != slot {
+                            info!("Slot: {}", slot);
+                            if current_slot != 0 {
+                                info!("Slot: {} - {:.2} MiB", slot, *bytes_per_slot.get(&current_slot).unwrap() as f64 / 1024.0 / 1024.0 );
+
+                                info!("Slot: {} - Updates: {}", slot, updates_per_slot.get(&current_slot).unwrap());
+
+                                let mut counters = count_updates_per_slot_account.iter()
+                                    .filter(|((slot, _pubkey), _)| slot == &current_slot)
+                                    .map(|((_slot, _pubkey), count)| *count as f64)
+                                    .sorted_by(|a, b| a.partial_cmp(b).unwrap())
+                                    .collect_vec();
+                                let count_histogram = histogram_percentiles::calculate_percentiles(&counters);
+                                info!("Count histogram: {}", count_histogram);
+
+
+                                let deltas = recent_slot_deltas.iter()
+                                    .map(|x| *x as f64)
+                                    .sorted_by(|a, b| a.partial_cmp(b).unwrap())
+                                    .collect_vec();
+                                let deltas_histogram = histogram_percentiles::calculate_percentiles(&deltas);
+                                info!("Deltas histogram: {}", deltas_histogram);
+
+                            }
+                            current_slot = slot;
+                        }
+
+                    }
+                    Some(UpdateOneof::Slot(update)) => {
+                        actual_slot = update.slot;
                     }
                     None => {}
                     _ => {}
diff --git a/src/lib.rs b/src/lib.rs
index 1ec5715..4baf31b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -11,8 +11,9 @@ pub mod channel_plugger;
 pub mod grpc_subscription_autoreconnect_streams;
 pub mod grpc_subscription_autoreconnect_tasks;
 pub mod grpcmultiplex_fastestwins;
-pub mod yellowstone_grpc_util;
 mod obfuscate;
+pub mod histogram_percentiles;
+pub mod yellowstone_grpc_util;
 
 // 1-based attempt counter
 type Attempt = u32;

From edd3a1a5439c780028dd4a1660b49e8b5e2e8bca Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 14:54:37 +0200
Subject: [PATCH 19/67] histogram

---
 src/histogram_percentiles.rs | 244 +++++++++++++++++++++++++++++++++++
 1 file changed, 244 insertions(+)
 create mode 100644 src/histogram_percentiles.rs

diff --git a/src/histogram_percentiles.rs b/src/histogram_percentiles.rs
new file mode 100644
index 0000000..09df7fd
--- /dev/null
+++ b/src/histogram_percentiles.rs
@@ -0,0 +1,244 @@
+use itertools::Itertools;
+use std::fmt::Display;
+use std::iter::zip;
+
+// #[derive(Clone, Copy, Debug, Default)]
+pub struct Point {
+    pub priority: f64,
+    pub value: f64,
+}
+
+impl From<(f64, f64)> for Point {
+    fn from((priority, cu_consumed): (f64, f64)) -> Self {
+        Point {
+            priority,
+            value: cu_consumed,
+        }
+    }
+}
+
+// #[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct HistValue {
+    pub percentile: f32,
+    pub value: f64,
+}
+
+/// `quantile` function is the same as the median if q=50, the same as the minimum if q=0 and the same as the maximum if q=100.
+
+pub fn calculate_percentiles(input: &[f64]) -> Percentiles {
+    if input.is_empty() {
+        // note: percentile for empty array is undefined
+        return Percentiles {
+            v: vec![],
+            p: vec![],
+        };
+    }
+
+    let is_monotonic = input.windows(2).all(|w| w[0] <= w[1]);
+    assert!(is_monotonic, "array of values must be sorted");
+
+    let p_step = 5;
+    let i_percentiles = (0..=100).step_by(p_step).collect_vec();
+
+    let mut bucket_values = Vec::with_capacity(i_percentiles.len());
+    let mut percentiles = Vec::with_capacity(i_percentiles.len());
+    for p in i_percentiles {
+        let value = {
+            let index = input.len() * p / 100;
+            let cap_index = index.min(input.len() - 1);
+            input[cap_index]
+        };
+
+        bucket_values.push(value);
+        percentiles.push(p as f32 / 100.0);
+    }
+
+    Percentiles {
+        v: bucket_values,
+        p: percentiles,
+    }
+}
+
+pub fn calculate_cummulative(values: &[Point]) -> PercentilesCummulative {
+    if values.is_empty() {
+        // note: percentile for empty array is undefined
+        return PercentilesCummulative {
+            bucket_values: vec![],
+            percentiles: vec![],
+        };
+    }
+
+    let is_monotonic = values.windows(2).all(|w| w[0].priority <= w[1].priority);
+    assert!(is_monotonic, "array of values must be sorted");
+
+    let value_sum: f64 = values.iter().map(|x| x.value).sum();
+    let mut agg: f64 = values[0].value;
+    let mut index = 0;
+    let p_step = 5;
+
+    let percentiles = (0..=100).step_by(p_step).map(|p| p as f64).collect_vec();
+
+    let dist = percentiles
+        .iter()
+        .map(|percentile| {
+            while agg < (value_sum * *percentile) / 100.0 {
+                index += 1;
+                agg += values[index].value;
+            }
+            let priority = values[index].priority;
+            HistValue {
+                percentile: *percentile as f32,
+                value: priority,
+            }
+        })
+        .collect_vec();
+
+    PercentilesCummulative {
+        bucket_values: dist.iter().map(|hv| hv.value).collect_vec(),
+        percentiles: dist.iter().map(|hv| hv.percentile / 100.0).collect_vec(),
+    }
+}
+
+pub struct Percentiles {
+    // value
+    pub v: Vec<f64>,
+    // percentile in range 0.0..1.0
+    pub p: Vec<f32>,
+}
+
+impl Display for Percentiles {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for i in 0..self.v.len() {
+            write!(f, "p{}=>{} ", self.p[i] * 100.0, self.v[i])?;
+        }
+        Ok(())
+    }
+}
+
+#[allow(dead_code)]
+impl Percentiles {
+    fn get_bucket_value(&self, percentile: f32) -> Option<f64> {
+        zip(&self.p, &self.v)
+            .find(|(&p, _v)| p == percentile)
+            .map(|(_p, &v)| v)
+    }
+}
+
+pub struct PercentilesCummulative {
+    pub bucket_values: Vec<f64>,
+    pub percentiles: Vec<f32>,
+}
+
+#[allow(dead_code)]
+impl PercentilesCummulative {
+    fn get_bucket_value(&self, percentile: f32) -> Option<f64> {
+        zip(&self.percentiles, &self.bucket_values)
+            .find(|(&p, _cu)| p == percentile)
+            .map(|(_p, &cu)| cu)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_calculate_percentiles() {
+        let mut values = vec![2.0, 4.0, 5.0, 3.0, 1.0];
+        values.sort_by_key(|&x| (x * 100.0) as i64);
+        let percentiles = calculate_percentiles(&values).v;
+        assert_eq!(percentiles[0], 1.0);
+        assert_eq!(percentiles[10], 3.0);
+        assert_eq!(percentiles[15], 4.0);
+        assert_eq!(percentiles[18], 5.0);
+        assert_eq!(percentiles[20], 5.0);
+    }
+
+    #[test]
+    fn test_calculate_percentiles_by_cu() {
+        // total of 20000 CU where consumed
+        let values = vec![Point::from((100.0, 10000.0)), Point::from((200.0, 10000.0))];
+        let PercentilesCummulative {
+            bucket_values: by_cu,
+            percentiles: by_cu_percentiles,
+            ..
+        } = calculate_cummulative(&values);
+        assert_eq!(by_cu_percentiles[10], 0.5);
+        assert_eq!(by_cu[10], 100.0); // need more than 100 to beat 50% of the CU
+        assert_eq!(by_cu[11], 200.0); // need more than 200 to beat 55% of the CU
+        assert_eq!(by_cu[20], 200.0); // need more than 200 to beat 100% of the CU
+    }
+
+    #[test]
+    fn test_empty_array() {
+        let values = vec![];
+        let percentiles = calculate_percentiles(&values).v;
+        // note: this is controversal
+        assert!(percentiles.is_empty());
+    }
+    #[test]
+    fn test_zeros() {
+        let values = vec![Point::from((0.0, 0.0)), Point::from((0.0, 0.0))];
+        let percentiles = calculate_cummulative(&values).bucket_values;
+        assert_eq!(percentiles[0], 0.0);
+    }
+
+    #[test]
+    fn test_statisticshowto() {
+        let values = vec![30.0, 33.0, 43.0, 53.0, 56.0, 67.0, 68.0, 72.0];
+        let percentiles = calculate_percentiles(&values);
+        assert_eq!(percentiles.v[5], 43.0);
+        assert_eq!(percentiles.p[5], 0.25);
+        assert_eq!(percentiles.get_bucket_value(0.25), Some(43.0));
+
+        let values = vec![
+            Point::from((30.0, 1.0)),
+            Point::from((33.0, 2.0)),
+            Point::from((43.0, 3.0)),
+            Point::from((53.0, 4.0)),
+            Point::from((56.0, 5.0)),
+            Point::from((67.0, 6.0)),
+            Point::from((68.0, 7.0)),
+            Point::from((72.0, 8.0)),
+        ];
+        let percentiles = calculate_cummulative(&values);
+        assert_eq!(percentiles.percentiles[20], 1.0);
+        assert_eq!(percentiles.bucket_values[20], 72.0);
+    }
+
+    #[test]
+    fn test_simple_non_integer_index() {
+        // Messwerte: 3 – 5 – 5 – 6 – 7 – 7 – 8 – 10 – 10
+        // In diesem Fall lautet es also 5.
+        let values = vec![3.0, 5.0, 5.0, 6.0, 7.0, 7.0, 8.0, 10.0, 10.0];
+
+        let percentiles = calculate_percentiles(&values);
+        assert_eq!(percentiles.p[4], 0.20);
+        assert_eq!(percentiles.v[5], 5.0);
+
+        let values = vec![
+            Point::from((3.0, 1.0)),
+            Point::from((5.0, 2.0)),
+            Point::from((5.0, 3.0)),
+            Point::from((6.0, 4.0)),
+            Point::from((7.0, 5.0)),
+            Point::from((7.0, 6.0)),
+            Point::from((8.0, 7.0)),
+            Point::from((10.0, 8.0)),
+            Point::from((10.0, 9.0)),
+        ];
+        let percentiles = calculate_cummulative(&values);
+        assert_eq!(percentiles.percentiles[19], 0.95);
+        assert_eq!(percentiles.percentiles[20], 1.0);
+        assert_eq!(percentiles.bucket_values[19], 10.0);
+        assert_eq!(percentiles.bucket_values[20], 10.0);
+    }
+
+    #[test]
+    fn test_large_list() {
+        let values = (0..1000).map(|i| i as f64).collect_vec();
+        let percentiles = calculate_percentiles(&values);
+        assert_eq!(percentiles.v[19], 950.0);
+        assert_eq!(percentiles.p[19], 0.95);
+    }
+}

From eb964e3492a3d847a7f0269c5db9c877b5855af5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 14:55:00 +0200
Subject: [PATCH 20/67] log deltas

---
 examples/bench_geyser_grpc_accounts.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 09dde99..55dc203 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -118,7 +118,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
 
                                 info!("Slot: {} - Updates: {}", slot, updates_per_slot.get(&current_slot).unwrap());
 
-                                let mut counters = count_updates_per_slot_account.iter()
+                                let counters = count_updates_per_slot_account.iter()
                                     .filter(|((slot, _pubkey), _)| slot == &current_slot)
                                     .map(|((_slot, _pubkey), count)| *count as f64)
                                     .sorted_by(|a, b| a.partial_cmp(b).unwrap())
@@ -132,6 +132,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                                     .sorted_by(|a, b| a.partial_cmp(b).unwrap())
                                     .collect_vec();
                                 let deltas_histogram = histogram_percentiles::calculate_percentiles(&deltas);
+                                info!("Deltas slots list: {:?}", recent_slot_deltas);
                                 info!("Deltas histogram: {}", deltas_histogram);
 
                             }

From 28c3041def9262f8c022f46808761ed321f4e731 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 15:25:46 +0200
Subject: [PATCH 21/67] log blocktime

---
 examples/bench_geyser_grpc_accounts.rs | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 55dc203..59555de 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -1,11 +1,12 @@
 use std::collections::{HashMap, VecDeque};
 use futures::{Stream, StreamExt};
 use log::info;
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Slot, UnixTimestamp};
 use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::pubkey::Pubkey;
 use std::env;
 use std::pin::pin;
+use std::time::{SystemTime, UNIX_EPOCH};
 use itertools::Itertools;
 use tokio::sync::mpsc::Receiver;
 
@@ -55,9 +56,9 @@ pub async fn main() {
         exit_notify.resubscribe(),
     );
 
-    let _slots_task = create_geyser_autoconnection_task_with_mpsc(
+    let _blocksmeta_task = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
-        GeyserFilter(CommitmentConfig::processed()).slots(),
+        GeyserFilter(CommitmentConfig::processed()).blocks_meta(),
         autoconnect_tx.clone(),
         exit_notify.resubscribe(),
     );
@@ -82,6 +83,8 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
         // slot from slot stream
         let mut actual_slot: Slot = 0;
 
+        // seconds since epoch
+        let mut block_time_per_slot = HashMap::<Slot, UnixTimestamp>::new();
         let mut recent_slot_deltas: VecDeque<i64> = VecDeque::with_capacity(RECENT_SLOTS_LIMIT);
 
         loop {
@@ -91,6 +94,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                         let account_info = update.account.unwrap();
                         let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
                         let slot = update.slot;
+                        let account_receive_time = get_epoch_sec();
 
                         if actual_slot != slot {
                             if actual_slot != 0 {
@@ -135,13 +139,19 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                                 info!("Deltas slots list: {:?}", recent_slot_deltas);
                                 info!("Deltas histogram: {}", deltas_histogram);
 
+                                if let Some(actual_block_time) = block_time_per_slot.get(&current_slot) {
+                                    let now = get_epoch_sec();
+                                    info!("Block time for slot {}: delta {} seconds", current_slot, *actual_block_time - now);
+                                }
+
                             }
                             current_slot = slot;
                         }
 
                     }
-                    Some(UpdateOneof::Slot(update)) => {
+                    Some(UpdateOneof::BlockMeta(update)) => {
                         actual_slot = update.slot;
+                        block_time_per_slot.insert(actual_slot, update.block_time.unwrap().timestamp);
                     }
                     None => {}
                     _ => {}
@@ -155,3 +165,10 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
         }
     });
 }
+
+fn get_epoch_sec() -> UnixTimestamp {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs() as UnixTimestamp
+}

From 2d65f2d68813e6d5eeef123d5d4002d256b58ae5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 15:27:41 +0200
Subject: [PATCH 22/67] flip delta time

---
 examples/bench_geyser_grpc_accounts.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 59555de..533bd1d 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -140,8 +140,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                                 info!("Deltas histogram: {}", deltas_histogram);
 
                                 if let Some(actual_block_time) = block_time_per_slot.get(&current_slot) {
-                                    let now = get_epoch_sec();
-                                    info!("Block time for slot {}: delta {} seconds", current_slot, *actual_block_time - now);
+                                    info!("Block time for slot {}: delta {} seconds", current_slot, account_receive_time - *actual_block_time);
                                 }
 
                             }

From 1e83353f37c4b131fea810c6067761934e359938 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 29 Apr 2024 15:34:00 +0200
Subject: [PATCH 23/67] document on slot delta

---
 examples/bench_geyser_grpc_accounts.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 533bd1d..b6e14b0 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -93,11 +93,13 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                     Some(UpdateOneof::Account(update)) => {
                         let account_info = update.account.unwrap();
                         let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
+                        // note: slot is referencing the block that is just built while the slot number reported from BlockMeta/Slot uses the slot after the block is built
                         let slot = update.slot;
                         let account_receive_time = get_epoch_sec();
 
                         if actual_slot != slot {
                             if actual_slot != 0 {
+                                // the perfect is value "-1"
                                 recent_slot_deltas.push_back((actual_slot as i64) - (slot as i64));
                                 if recent_slot_deltas.len() > RECENT_SLOTS_LIMIT {
                                     recent_slot_deltas.pop_front();

From 8e5b1b8f7340491922a55396f8eecf51e5497763 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 30 Apr 2024 12:38:39 +0200
Subject: [PATCH 24/67] log update span

---
 examples/bench_geyser_grpc_accounts.rs | 45 ++++++++++++++++++--------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index b6e14b0..0f3ecec 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -69,7 +69,7 @@ pub async fn main() {
     sleep(Duration::from_secs(1800)).await;
 }
 
-
+// note: this keeps track of lot of data and might blow up memory
 fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
     const RECENT_SLOTS_LIMIT: usize = 30;
 
@@ -77,30 +77,33 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
 
         let mut bytes_per_slot = HashMap::<Slot, usize>::new();
         let mut updates_per_slot = HashMap::<Slot, usize>::new();
-        let mut count_updates_per_slot_account = HashMap::<(Slot, Pubkey), usize>::new();
+        let mut wallclock_updates_per_slot_account = HashMap::<(Slot, Pubkey), Vec<SystemTime>>::new();
         // slot written by account update
         let mut current_slot: Slot = 0;
         // slot from slot stream
-        let mut actual_slot: Slot = 0;
+        let mut slot_just_completed: Slot = 0;
 
         // seconds since epoch
         let mut block_time_per_slot = HashMap::<Slot, UnixTimestamp>::new();
+        // wall clock time of block completion (i.e. processed) reported by the block meta stream
+        let mut block_completion_notification_time_per_slot = HashMap::<Slot, SystemTime>::new();
         let mut recent_slot_deltas: VecDeque<i64> = VecDeque::with_capacity(RECENT_SLOTS_LIMIT);
 
         loop {
             match geyser_messages_rx.recv().await {
                 Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
                     Some(UpdateOneof::Account(update)) => {
+                        let now = SystemTime::now();
                         let account_info = update.account.unwrap();
                         let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
                         // note: slot is referencing the block that is just built while the slot number reported from BlockMeta/Slot uses the slot after the block is built
                         let slot = update.slot;
                         let account_receive_time = get_epoch_sec();
 
-                        if actual_slot != slot {
-                            if actual_slot != 0 {
+                        if slot_just_completed != slot {
+                            if slot_just_completed != 0 {
                                 // the perfect is value "-1"
-                                recent_slot_deltas.push_back((actual_slot as i64) - (slot as i64));
+                                recent_slot_deltas.push_back((slot_just_completed as i64) - (slot as i64));
                                 if recent_slot_deltas.len() > RECENT_SLOTS_LIMIT {
                                     recent_slot_deltas.pop_front();
                                 }
@@ -113,9 +116,9 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                         updates_per_slot.entry(slot)
                             .and_modify(|entry| *entry += 1)
                             .or_insert(1);
-                        count_updates_per_slot_account.entry((slot, account_pk))
-                            .and_modify(|entry| *entry += 1)
-                            .or_insert(1);
+                        wallclock_updates_per_slot_account.entry((slot, account_pk))
+                            .and_modify(|entry| entry.push(now))
+                            .or_insert(vec![now]);
 
                         if current_slot != slot {
                             info!("Slot: {}", slot);
@@ -124,15 +127,14 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
 
                                 info!("Slot: {} - Updates: {}", slot, updates_per_slot.get(&current_slot).unwrap());
 
-                                let counters = count_updates_per_slot_account.iter()
+                                let counters = wallclock_updates_per_slot_account.iter()
                                     .filter(|((slot, _pubkey), _)| slot == &current_slot)
-                                    .map(|((_slot, _pubkey), count)| *count as f64)
+                                    .map(|((_slot, _pubkey), updates)| updates.len() as f64)
                                     .sorted_by(|a, b| a.partial_cmp(b).unwrap())
                                     .collect_vec();
                                 let count_histogram = histogram_percentiles::calculate_percentiles(&counters);
                                 info!("Count histogram: {}", count_histogram);
 
-
                                 let deltas = recent_slot_deltas.iter()
                                     .map(|x| *x as f64)
                                     .sorted_by(|a, b| a.partial_cmp(b).unwrap())
@@ -145,14 +147,29 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                                     info!("Block time for slot {}: delta {} seconds", current_slot, account_receive_time - *actual_block_time);
                                 }
 
+                                let wallclock_minmax = wallclock_updates_per_slot_account.iter()
+                                    .filter(|((slot, _pubkey), _)| slot == &current_slot)
+                                    .flat_map(|((_slot, _pubkey), updates)| updates)
+                                    .minmax();
+                                if let Some((min, max)) = wallclock_minmax.into_option() {
+                                    info!("Wallclock timestamp between first and last account update received for slot {}: {:.2}s",
+                                        current_slot,
+                                        max.duration_since(*min).unwrap().as_secs_f64()
+                                    );
+                                }
+
+
                             }
                             current_slot = slot;
                         }
 
                     }
                     Some(UpdateOneof::BlockMeta(update)) => {
-                        actual_slot = update.slot;
-                        block_time_per_slot.insert(actual_slot, update.block_time.unwrap().timestamp);
+                        let now = SystemTime::now();
+                        // completed depends on commitment level which is processed ATM
+                        slot_just_completed = update.slot;
+                        block_time_per_slot.insert(slot_just_completed, update.block_time.unwrap().timestamp);
+                        block_completion_notification_time_per_slot.insert(slot_just_completed, now);
                     }
                     None => {}
                     _ => {}

From a6cb6bb17f2d8adc4e62cd8f8baae8668d753145 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 2 May 2024 12:02:52 +0200
Subject: [PATCH 25/67]  mock correct size distribution

---
 examples/bench_geyser_grpc_accounts.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 0f3ecec..1556f83 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -123,9 +123,9 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                         if current_slot != slot {
                             info!("Slot: {}", slot);
                             if current_slot != 0 {
-                                info!("Slot: {} - {:.2} MiB", slot, *bytes_per_slot.get(&current_slot).unwrap() as f64 / 1024.0 / 1024.0 );
+                                info!("Slot: {} - account data transferred: {:.2} MiB", slot, *bytes_per_slot.get(&current_slot).unwrap() as f64 / 1024.0 / 1024.0 );
 
-                                info!("Slot: {} - Updates: {}", slot, updates_per_slot.get(&current_slot).unwrap());
+                                info!("Slot: {} - num of update messages: {}", slot, updates_per_slot.get(&current_slot).unwrap());
 
                                 let counters = wallclock_updates_per_slot_account.iter()
                                     .filter(|((slot, _pubkey), _)| slot == &current_slot)

From 39199b842a89edf2f7039472d69cc752142ad283 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 3 May 2024 14:52:29 +0200
Subject: [PATCH 26/67] stream token  accounts

---
 examples/stream_token_accounts.rs | 197 ++++++++++++++++++++++++++++++
 1 file changed, 197 insertions(+)
 create mode 100644 examples/stream_token_accounts.rs

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
new file mode 100644
index 0000000..ab70157
--- /dev/null
+++ b/examples/stream_token_accounts.rs
@@ -0,0 +1,197 @@
+use std::collections::HashMap;
+use futures::{Stream, StreamExt};
+use log::info;
+use solana_sdk::clock::Slot;
+use solana_sdk::commitment_config::CommitmentConfig;
+use std::env;
+use std::pin::pin;
+use std::str::FromStr;
+use solana_sdk::pubkey::Pubkey;
+
+use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
+use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
+use tokio::time::{sleep, Duration};
+use tracing::warn;
+use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
+use yellowstone_grpc_proto::prost::Message as _;
+
+#[allow(dead_code)]
+fn start_example_blockmini_consumer(
+    multiplex_stream: impl Stream<Item = BlockMini> + Send + 'static,
+) {
+    tokio::spawn(async move {
+        let mut blockmeta_stream = pin!(multiplex_stream);
+        while let Some(mini) = blockmeta_stream.next().await {
+            info!(
+                "emitted block mini #{}@{} with {} bytes from multiplexer",
+                mini.slot, mini.commitment_config.commitment, mini.blocksize
+            );
+        }
+    });
+}
+
+pub struct BlockMini {
+    pub blocksize: usize,
+    pub slot: Slot,
+    pub commitment_config: CommitmentConfig,
+}
+
+struct BlockMiniExtractor(CommitmentConfig);
+
+impl FromYellowstoneExtractor for BlockMiniExtractor {
+    type Target = BlockMini;
+    fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(Slot, Self::Target)> {
+        match update.update_oneof {
+            Some(UpdateOneof::Block(update_block_message)) => {
+                let blocksize = update_block_message.encoded_len();
+                let slot = update_block_message.slot;
+                let mini = BlockMini {
+                    blocksize,
+                    slot,
+                    commitment_config: self.0,
+                };
+                Some((slot, mini))
+            }
+            Some(UpdateOneof::BlockMeta(update_blockmeta_message)) => {
+                let blocksize = update_blockmeta_message.encoded_len();
+                let slot = update_blockmeta_message.slot;
+                let mini = BlockMini {
+                    blocksize,
+                    slot,
+                    commitment_config: self.0,
+                };
+                Some((slot, mini))
+            }
+            _ => None,
+        }
+    }
+}
+
+#[tokio::main]
+pub async fn main() {
+    // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
+    tracing_subscriber::fmt::init();
+    // console_subscriber::init();
+
+    let COMMITMENT_LEVEL = CommitmentConfig::processed();
+    let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green");
+    let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok();
+
+    info!(
+        "Using grpc source on {} ({})",
+        grpc_addr_green,
+        grpc_x_token_green.is_some()
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(25),
+        request_timeout: Duration::from_secs(25),
+        subscribe_timeout: Duration::from_secs(25),
+        receive_timeout: Duration::from_secs(25),
+    };
+
+    let config = GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
+
+    info!("Write Block stream..");
+
+    let green_stream = create_geyser_reconnecting_stream(
+        config.clone(),
+        // GeyserFilter(COMMITMENT_LEVEL).accounts(),
+        token_accounts(),
+    );
+
+    let blue_stream = create_geyser_reconnecting_stream(
+        config.clone(),
+        GeyserFilter(COMMITMENT_LEVEL).blocks_and_txs(),
+    );
+
+    tokio::spawn(async move {
+        let token_pk = Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap();
+        let mut green_stream = pin!(green_stream);
+        while let Some(message) = green_stream.next().await {
+            match message {
+                Message::GeyserSubscribeUpdate(subscriber_update) => {
+                    match subscriber_update.update_oneof {
+                        Some(UpdateOneof::Account(update)) => {
+                            let account_info = update.account.unwrap();
+                            let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
+
+                            // if account_pk == token_pk {
+                                info!("got account update (green)!!! {} - {:?} - {} bytes",
+                                    update.slot, account_pk, account_info.data.len());
+                            // }
+                            let bytes: [u8; 32] =
+                                account_pk.to_bytes();
+                        }
+                        _ => {}
+                    }
+                }
+                Message::Connecting(attempt) => {
+                    warn!("Connection attempt: {}", attempt);
+                }
+            }
+        }
+        warn!("Stream aborted");
+    });
+
+    tokio::spawn(async move {
+        let mut blue_stream = pin!(blue_stream);
+        let extractor = BlockMiniExtractor(COMMITMENT_LEVEL);
+        while let Some(message) = blue_stream.next().await {
+            match message {
+                Message::GeyserSubscribeUpdate(subscriber_update) => {
+                    let mapped = extractor.map_yellowstone_update(*subscriber_update);
+                    if let Some((slot, block_mini)) = mapped {
+                        info!("got update (blue)!!! block: {} - {} bytes", slot, block_mini.blocksize);
+                    }
+                }
+                Message::Connecting(attempt) => {
+                    warn!("Connection attempt: {}", attempt);
+                }
+            }
+        }
+        warn!("Stream aborted");
+    });
+
+    // "infinite" sleep
+    sleep(Duration::from_secs(1800)).await;
+}
+
+fn map_block_update(update: SubscribeUpdate) -> Option<Slot> {
+    match update.update_oneof {
+        Some(UpdateOneof::Block(update_block_message)) => {
+            let slot = update_block_message.slot;
+            Some(slot)
+        }
+        _ => None,
+    }
+}
+
+pub fn token_accounts() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![
+                "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA".to_string(),
+            ],
+            filters: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        slots: HashMap::new(),
+        accounts: accounts_subs,
+        transactions: HashMap::new(),
+        entry: Default::default(),
+        blocks: Default::default(),
+        blocks_meta: HashMap::new(),
+        commitment: None,
+        accounts_data_slice: Default::default(),
+        ping: None,
+    }
+}
+

From 2cef5ef87b86f4bfdef10f123fe1b9b699490a6c Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 3 May 2024 16:07:12 +0200
Subject: [PATCH 27/67] token account

---
 examples/stream_token_accounts.rs | 132 ++++++++----------------------
 1 file changed, 34 insertions(+), 98 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index ab70157..4f71e87 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -6,6 +6,7 @@ use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
 use std::pin::pin;
 use std::str::FromStr;
+use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType};
 use solana_sdk::pubkey::Pubkey;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
@@ -17,65 +18,12 @@ use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
 
-#[allow(dead_code)]
-fn start_example_blockmini_consumer(
-    multiplex_stream: impl Stream<Item = BlockMini> + Send + 'static,
-) {
-    tokio::spawn(async move {
-        let mut blockmeta_stream = pin!(multiplex_stream);
-        while let Some(mini) = blockmeta_stream.next().await {
-            info!(
-                "emitted block mini #{}@{} with {} bytes from multiplexer",
-                mini.slot, mini.commitment_config.commitment, mini.blocksize
-            );
-        }
-    });
-}
-
-pub struct BlockMini {
-    pub blocksize: usize,
-    pub slot: Slot,
-    pub commitment_config: CommitmentConfig,
-}
-
-struct BlockMiniExtractor(CommitmentConfig);
-
-impl FromYellowstoneExtractor for BlockMiniExtractor {
-    type Target = BlockMini;
-    fn map_yellowstone_update(&self, update: SubscribeUpdate) -> Option<(Slot, Self::Target)> {
-        match update.update_oneof {
-            Some(UpdateOneof::Block(update_block_message)) => {
-                let blocksize = update_block_message.encoded_len();
-                let slot = update_block_message.slot;
-                let mini = BlockMini {
-                    blocksize,
-                    slot,
-                    commitment_config: self.0,
-                };
-                Some((slot, mini))
-            }
-            Some(UpdateOneof::BlockMeta(update_blockmeta_message)) => {
-                let blocksize = update_blockmeta_message.encoded_len();
-                let slot = update_blockmeta_message.slot;
-                let mini = BlockMini {
-                    blocksize,
-                    slot,
-                    commitment_config: self.0,
-                };
-                Some((slot, mini))
-            }
-            _ => None,
-        }
-    }
-}
-
 #[tokio::main]
 pub async fn main() {
     // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
     tracing_subscriber::fmt::init();
     // console_subscriber::init();
 
-    let COMMITMENT_LEVEL = CommitmentConfig::processed();
     let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green");
     let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok();
 
@@ -98,30 +46,47 @@ pub async fn main() {
 
     let green_stream = create_geyser_reconnecting_stream(
         config.clone(),
-        // GeyserFilter(COMMITMENT_LEVEL).accounts(),
         token_accounts(),
     );
 
-    let blue_stream = create_geyser_reconnecting_stream(
-        config.clone(),
-        GeyserFilter(COMMITMENT_LEVEL).blocks_and_txs(),
-    );
 
     tokio::spawn(async move {
-        let token_pk = Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap();
         let mut green_stream = pin!(green_stream);
         while let Some(message) = green_stream.next().await {
             match message {
                 Message::GeyserSubscribeUpdate(subscriber_update) => {
                     match subscriber_update.update_oneof {
                         Some(UpdateOneof::Account(update)) => {
-                            let account_info = update.account.unwrap();
-                            let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
-
-                            // if account_pk == token_pk {
-                                info!("got account update (green)!!! {} - {:?} - {} bytes",
-                                    update.slot, account_pk, account_info.data.len());
-                            // }
+                            let account = update.account.unwrap();
+                            let account_pk = Pubkey::try_from(account.pubkey).unwrap();
+
+                            let account_type = parse_token(&account.data, Some(6)).unwrap();
+                            match account_type {
+                                TokenAccountType::Account(account_ui) => {
+                                    // UiTokenAccount {
+                                    //   mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
+                                    //   owner: "7XDMxfmzmL2Hqhh8ABp4Byc5UEsMWfQdWo6r5vEVQNb8",
+                                    //   token_amount: UiTokenAmount {
+                                    //     ui_amount: Some(0.0), decimals: 6, amount: "0", ui_amount_string: "0"
+                                    //   },
+                                    //   delegate: None,
+                                    //   state: Initialized,
+                                    //   is_native: false,
+                                    //   rent_exempt_reserve: None,
+                                    //   delegated_amount: None,
+                                    //   close_authority: None,
+                                    //   extensions: []
+                                    // }
+                                    info!("it's an Account: {:?}", account_ui);
+                                }
+                                TokenAccountType::Mint(mint) => {
+                                    // not interesting
+                                }
+                                TokenAccountType::Multisig(_) => {}
+                            }
+
+                            info!("got account update (green)!!! {} - {:?} - {} bytes",
+                                update.slot, account_pk, account.data.len());
                             let bytes: [u8; 32] =
                                 account_pk.to_bytes();
                         }
@@ -136,52 +101,23 @@ pub async fn main() {
         warn!("Stream aborted");
     });
 
-    tokio::spawn(async move {
-        let mut blue_stream = pin!(blue_stream);
-        let extractor = BlockMiniExtractor(COMMITMENT_LEVEL);
-        while let Some(message) = blue_stream.next().await {
-            match message {
-                Message::GeyserSubscribeUpdate(subscriber_update) => {
-                    let mapped = extractor.map_yellowstone_update(*subscriber_update);
-                    if let Some((slot, block_mini)) = mapped {
-                        info!("got update (blue)!!! block: {} - {} bytes", slot, block_mini.blocksize);
-                    }
-                }
-                Message::Connecting(attempt) => {
-                    warn!("Connection attempt: {}", attempt);
-                }
-            }
-        }
-        warn!("Stream aborted");
-    });
-
     // "infinite" sleep
     sleep(Duration::from_secs(1800)).await;
 }
 
-fn map_block_update(update: SubscribeUpdate) -> Option<Slot> {
-    match update.update_oneof {
-        Some(UpdateOneof::Block(update_block_message)) => {
-            let slot = update_block_message.slot;
-            Some(slot)
-        }
-        _ => None,
-    }
-}
-
 pub fn token_accounts() -> SubscribeRequest {
     let mut accounts_subs = HashMap::new();
     accounts_subs.insert(
         "client".to_string(),
         SubscribeRequestFilterAccounts {
             account: vec![],
-            owner: vec![
-                "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA".to_string(),
-            ],
+            owner:
+                spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
         },
     );
 
+
     SubscribeRequest {
         slots: HashMap::new(),
         accounts: accounts_subs,

From 1b536cbd14e3988c7be7ac791596b7e05925d3b3 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 3 May 2024 17:23:41 +0200
Subject: [PATCH 28/67] build map of mint+owner

---
 Cargo.lock                        | 15 +++++++
 Cargo.toml                        |  4 ++
 examples/stream_token_accounts.rs | 71 +++++++++++++++++++++++++------
 3 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 470a53e..117d128 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -913,6 +913,19 @@ dependencies = [
  "syn 2.0.58",
 ]
 
+[[package]]
+name = "dashmap"
+version = "5.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+dependencies = [
+ "cfg-if",
+ "hashbrown 0.14.3",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
 [[package]]
 name = "derivation-path"
 version = "0.2.0"
@@ -1256,11 +1269,13 @@ dependencies = [
  "base64 0.21.7",
  "bincode",
  "csv",
+ "dashmap",
  "derive_more",
  "futures",
  "itertools 0.10.5",
  "log",
  "merge-streams",
+ "solana-account-decoder",
  "solana-logger",
  "solana-sdk",
  "tokio",
diff --git a/Cargo.toml b/Cargo.toml
index 95ca69d..e465c08 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,8 +31,12 @@ bincode = "1.3.3"
 
 csv = "1.3.0"
 
+dashmap = "5.5.3"
+
 tonic-health = "0.10.2"
 
 [dev-dependencies]
 tracing-subscriber = "0.3.16"
 solana-logger = "1"
+solana-account-decoder = "1"
+
diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index 4f71e87..db4c0b6 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -1,18 +1,22 @@
 use std::collections::HashMap;
 use futures::{Stream, StreamExt};
-use log::info;
+use log::{debug, info, trace};
 use solana_sdk::clock::Slot;
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
 use std::pin::pin;
 use std::str::FromStr;
-use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType};
+use std::sync::Arc;
+use dashmap::DashMap;
+use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType, UiTokenAccount};
+use solana_account_decoder::parse_token::UiAccountState::Initialized;
 use solana_sdk::pubkey::Pubkey;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
 use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
 use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
 use tokio::time::{sleep, Duration};
+use tracing::field::debug;
 use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
@@ -49,6 +53,11 @@ pub async fn main() {
         token_accounts(),
     );
 
+    // owner x mint -> amount
+    let token_account_by_ownermint: Arc<DashMap<Pubkey, DashMap<Pubkey, UiTokenAccount>>> = Arc::new(DashMap::with_capacity(10000));
+    let token_account_by_ownermint_read = token_account_by_ownermint.clone();
+    let token_account_by_ownermint = token_account_by_ownermint.clone();
+
 
     tokio::spawn(async move {
         let mut green_stream = pin!(green_stream);
@@ -59,15 +68,18 @@ pub async fn main() {
                         Some(UpdateOneof::Account(update)) => {
                             let account = update.account.unwrap();
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
+                            trace!("got account update (green)!!! {} - {:?} - {} bytes",
+                                update.slot, account_pk, account.data.len());
 
-                            let account_type = parse_token(&account.data, Some(6)).unwrap();
-                            match account_type {
-                                TokenAccountType::Account(account_ui) => {
+                            match parse_token(&account.data, Some(6)) {
+                                Ok(TokenAccountType::Account(account_ui)) => {
                                     // UiTokenAccount {
                                     //   mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
-                                    //   owner: "7XDMxfmzmL2Hqhh8ABp4Byc5UEsMWfQdWo6r5vEVQNb8",
+                                    //   owner: "9un5wqE3q4oCjyrDkwsdD48KteCJitQX5978Vh7KKxHo",
                                     //   token_amount: UiTokenAmount {
-                                    //     ui_amount: Some(0.0), decimals: 6, amount: "0", ui_amount_string: "0"
+                                    //     ui_amount: Some(8229995.070667),
+                                    //     decimals: 6, amount: "8229995070667",
+                                    //     ui_amount_string: "8229995.070667"
                                     //   },
                                     //   delegate: None,
                                     //   state: Initialized,
@@ -77,16 +89,27 @@ pub async fn main() {
                                     //   close_authority: None,
                                     //   extensions: []
                                     // }
-                                    info!("it's an Account: {:?}", account_ui);
+                                    if matches!(account_ui.state, Initialized) {
+                                        let owner = Pubkey::from_str(&account_ui.owner).unwrap();
+                                        let mint = Pubkey::from_str(&account_ui.mint).unwrap();
+                                        // 6 decimals as requested
+                                        let amount = &account_ui.token_amount.amount;
+                                        trace!("update balance for mint {} of owner {}: {}", mint, owner, amount);
+                                        token_account_by_ownermint.entry(owner)
+                                            .or_insert_with(DashMap::new)
+                                            .insert(mint, account_ui);
+                                    }
                                 }
-                                TokenAccountType::Mint(mint) => {
+                                Ok(TokenAccountType::Mint(mint)) => {
                                     // not interesting
                                 }
-                                TokenAccountType::Multisig(_) => {}
+                                Ok(TokenAccountType::Multisig(_)) => {}
+                                Err(parse_error) => {
+                                    debug!("Could not parse account {} - {}", account_pk, parse_error);
+                                }
                             }
 
-                            info!("got account update (green)!!! {} - {:?} - {} bytes",
-                                update.slot, account_pk, account.data.len());
+
                             let bytes: [u8; 32] =
                                 account_pk.to_bytes();
                         }
@@ -101,6 +124,30 @@ pub async fn main() {
         warn!("Stream aborted");
     });
 
+
+    tokio::spawn(async move {
+
+        loop {
+            let mut total = 0;
+            for accounts_by_mint in token_account_by_ownermint_read.iter() {
+                for token_account_mint in accounts_by_mint.iter() {
+                    total += 1;
+                    let (owner, mint, account) = (accounts_by_mint.key(), token_account_mint.key(), token_account_mint.value());
+                    debug!("{} - {} - {}", owner, mint, account.token_amount.ui_amount_string);
+                    // info!("xx {} - {} - {}", foo.key(), bar.key(), bar.value().token_amount.ui_amount_string);
+                }
+                // for (mint, account) in mint_to_amount.iter() {
+                //     total += 1;
+                //     debug!("{} - {} - {} - {}", owner, mint, account.token_amount.ui_amount_string, account.token_amount.amount);
+                // }
+            }
+            debug!("Total: {}", total);
+            sleep(Duration::from_secs(1)).await;
+        }
+
+    });
+
+
     // "infinite" sleep
     sleep(Duration::from_secs(1800)).await;
 }

From e278c14346059df3fb10017a4f8de3dce1dad126 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 3 May 2024 18:03:42 +0200
Subject: [PATCH 29/67] build simple map

---
 examples/stream_token_accounts.rs | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index db4c0b6..b2c6686 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -94,7 +94,10 @@ pub async fn main() {
                                         let mint = Pubkey::from_str(&account_ui.mint).unwrap();
                                         // 6 decimals as requested
                                         let amount = &account_ui.token_amount.amount;
-                                        trace!("update balance for mint {} of owner {}: {}", mint, owner, amount);
+                                        // groovie wallet
+                                        if account_ui.owner.starts_with("66fEFnKy") {
+                                            info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
+                                        }
                                         token_account_by_ownermint.entry(owner)
                                             .or_insert_with(DashMap::new)
                                             .insert(mint, account_ui);
@@ -105,7 +108,7 @@ pub async fn main() {
                                 }
                                 Ok(TokenAccountType::Multisig(_)) => {}
                                 Err(parse_error) => {
-                                    debug!("Could not parse account {} - {}", account_pk, parse_error);
+                                    trace!("Could not parse account {} - {}", account_pk, parse_error);
                                 }
                             }
 
@@ -134,15 +137,10 @@ pub async fn main() {
                     total += 1;
                     let (owner, mint, account) = (accounts_by_mint.key(), token_account_mint.key(), token_account_mint.value());
                     debug!("{} - {} - {}", owner, mint, account.token_amount.ui_amount_string);
-                    // info!("xx {} - {} - {}", foo.key(), bar.key(), bar.value().token_amount.ui_amount_string);
                 }
-                // for (mint, account) in mint_to_amount.iter() {
-                //     total += 1;
-                //     debug!("{} - {} - {} - {}", owner, mint, account.token_amount.ui_amount_string, account.token_amount.amount);
-                // }
             }
-            debug!("Total: {}", total);
-            sleep(Duration::from_secs(1)).await;
+            info!("Total owner x mint entries in cache map: {}", total);
+            sleep(Duration::from_millis(50)).await;
         }
 
     });

From f60e9f2a3b036e7011b385842f28498decd158d2 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 6 May 2024 16:25:10 +0200
Subject: [PATCH 30/67] timestamp tagging

---
 examples/stream_token_accounts.rs | 38 ++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index b2c6686..77df5e5 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -22,6 +22,9 @@ use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
 
+
+const ENABLE_TIMESTAMP_TAGGING: bool = true;
+
 #[tokio::main]
 pub async fn main() {
     // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
@@ -71,6 +74,11 @@ pub async fn main() {
                             trace!("got account update (green)!!! {} - {:?} - {} bytes",
                                 update.slot, account_pk, account.data.len());
 
+                            if ENABLE_TIMESTAMP_TAGGING {
+                                let since_the_epoch = std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH).expect("Time went backwards");
+                                info!("got account update: write_version={};timestamp_us={};slot={}", account.write_version, since_the_epoch.as_micros(), update.slot);
+                            }
+
                             match parse_token(&account.data, Some(6)) {
                                 Ok(TokenAccountType::Account(account_ui)) => {
                                     // UiTokenAccount {
@@ -89,19 +97,23 @@ pub async fn main() {
                                     //   close_authority: None,
                                     //   extensions: []
                                     // }
-                                    if matches!(account_ui.state, Initialized) {
-                                        let owner = Pubkey::from_str(&account_ui.owner).unwrap();
-                                        let mint = Pubkey::from_str(&account_ui.mint).unwrap();
-                                        // 6 decimals as requested
-                                        let amount = &account_ui.token_amount.amount;
-                                        // groovie wallet
-                                        if account_ui.owner.starts_with("66fEFnKy") {
-                                            info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
-                                        }
-                                        token_account_by_ownermint.entry(owner)
-                                            .or_insert_with(DashMap::new)
-                                            .insert(mint, account_ui);
+                                    // all different states are covered
+                                    // is_native: both true+false are sent
+                                    assert_eq!(account.executable, false);
+                                    assert_eq!(account.rent_epoch, u64::MAX);
+
+                                    let owner = Pubkey::from_str(&account_ui.owner).unwrap();
+                                    let mint = Pubkey::from_str(&account_ui.mint).unwrap();
+                                    // 6 decimals as requested
+                                    let amount = &account_ui.token_amount.amount;
+                                    // groovie wallet
+                                    if account_ui.owner.starts_with("66fEFnKy") {
+                                        info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
                                     }
+                                    token_account_by_ownermint.entry(owner)
+                                        .or_insert_with(DashMap::new)
+                                        .insert(mint, account_ui);
+
                                 }
                                 Ok(TokenAccountType::Mint(mint)) => {
                                     // not interesting
@@ -136,7 +148,7 @@ pub async fn main() {
                 for token_account_mint in accounts_by_mint.iter() {
                     total += 1;
                     let (owner, mint, account) = (accounts_by_mint.key(), token_account_mint.key(), token_account_mint.value());
-                    debug!("{} - {} - {}", owner, mint, account.token_amount.ui_amount_string);
+                    // debug!("{} - {} - {}", owner, mint, account.token_amount.ui_amount_string);
                 }
             }
             info!("Total owner x mint entries in cache map: {}", total);

From e4bb528237ab0e0027832a0c62e2ef05f01b581b Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 6 May 2024 20:56:51 +0200
Subject: [PATCH 31/67] configure logger

---
 Cargo.lock                        | 1 +
 Cargo.toml                        | 1 +
 examples/stream_token_accounts.rs | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 117d128..88641ef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1275,6 +1275,7 @@ dependencies = [
  "itertools 0.10.5",
  "log",
  "merge-streams",
+ "regex",
  "solana-account-decoder",
  "solana-logger",
  "solana-sdk",
diff --git a/Cargo.toml b/Cargo.toml
index e465c08..1f8bd06 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,6 +34,7 @@ csv = "1.3.0"
 dashmap = "5.5.3"
 
 tonic-health = "0.10.2"
+regex = "1.10.4"
 
 [dev-dependencies]
 tracing-subscriber = "0.3.16"
diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index 77df5e5..531e753 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -152,7 +152,7 @@ pub async fn main() {
                 }
             }
             info!("Total owner x mint entries in cache map: {}", total);
-            sleep(Duration::from_millis(50)).await;
+            sleep(Duration::from_millis(1500)).await;
         }
 
     });

From 09d470046abbb6f399ee3b239ec1a89f4d96b8bd Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 7 May 2024 10:14:47 +0200
Subject: [PATCH 32/67] parse logs

---
 examples/parse_timestamp_tagged_logs.rs | 66 +++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 examples/parse_timestamp_tagged_logs.rs

diff --git a/examples/parse_timestamp_tagged_logs.rs b/examples/parse_timestamp_tagged_logs.rs
new file mode 100644
index 0000000..ca11dcd
--- /dev/null
+++ b/examples/parse_timestamp_tagged_logs.rs
@@ -0,0 +1,66 @@
+use std::collections::HashMap;
+use std::fs::File;
+use std::io;
+use std::io::BufRead;
+use std::path::{Path, PathBuf};
+use regex::Regex;
+
+pub fn parse_log_entry_subscriber(log_entry: &str) -> HashMap<String, String> {
+    let re = Regex::new(r".*got account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
+    let caps = re.captures(log_entry).unwrap();
+
+    let mut result = HashMap::new();
+    result.insert("write_version".to_string(), caps["write_version"].to_string());
+    result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
+    result.insert("slot".to_string(), caps["slot"].to_string());
+
+    result
+}
+
+pub fn parse_log_entry_source(log_entry: &str) -> HashMap<String, String> {
+    let re = Regex::new(r".*account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
+    let caps = re.captures(log_entry).unwrap();
+
+    let mut result = HashMap::new();
+    result.insert("write_version".to_string(), caps["write_version"].to_string());
+    result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
+    result.insert("slot".to_string(), caps["slot"].to_string());
+
+    result
+}
+
+
+
+fn read_subscriber_log(log_file: PathBuf) {
+
+    let file = File::open(log_file).expect("file must exist");
+    let reader = io::BufReader::new(file);
+    for line in reader.lines().take(10) {
+        let line = line.expect("must be parsable");
+        let parsed = parse_log_entry_subscriber(&line);
+        println!("{:?}", parsed);
+    }
+
+}
+
+fn read_source_log(log_file: PathBuf) {
+    let file = File::open(log_file).expect("file must exist");
+    let reader = io::BufReader::new(file);
+    for line in reader.lines().take(10) {
+        let line = line.expect("must be parsable");
+        let parsed = parse_log_entry_source(&line);
+        println!("{:?}", parsed);
+    }
+}
+
+
+pub fn main() {
+
+
+    read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
+
+    read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
+
+
+
+}

From b64f7002db606ab516ce410746b7744db6120921 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 7 May 2024 10:54:06 +0200
Subject: [PATCH 33/67] args

---
 accounts-testnet.csv | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 accounts-testnet.csv

diff --git a/accounts-testnet.csv b/accounts-testnet.csv
new file mode 100644
index 0000000..e69de29

From 1b5d4f458dde4a88c86d667c292dc24342429d32 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 7 May 2024 10:59:11 +0200
Subject: [PATCH 34/67] provide logs as args

---
 Cargo.lock                              | 123 +++++++++++++++++++++-
 Cargo.toml                              |   1 +
 examples/parse_timestamp_tagged_logs.rs | 134 ++++++++++++++++++++----
 3 files changed, 235 insertions(+), 23 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 88641ef..e760222 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -126,6 +126,55 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.82"
@@ -695,6 +744,54 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "clap"
+version = "4.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+ "once_cell",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "bitflags 1.3.2",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.58",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+
 [[package]]
 name = "combine"
 version = "3.8.1"
@@ -1268,6 +1365,7 @@ dependencies = [
  "async-stream",
  "base64 0.21.7",
  "bincode",
+ "clap",
  "csv",
  "dashmap",
  "derive_more",
@@ -1363,6 +1461,12 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1585,6 +1689,17 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+dependencies = [
+ "hermit-abi 0.3.9",
+ "libc",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -2216,7 +2331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1"
 dependencies = [
  "bytes",
- "heck",
+ "heck 0.5.0",
  "itertools 0.12.1",
  "log",
  "multimap",
@@ -3858,6 +3973,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "valuable"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 1f8bd06..ddce63b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,6 +35,7 @@ dashmap = "5.5.3"
 
 tonic-health = "0.10.2"
 regex = "1.10.4"
+clap = { version = "4.2", features = ["derive"] }
 
 [dev-dependencies]
 tracing-subscriber = "0.3.16"
diff --git a/examples/parse_timestamp_tagged_logs.rs b/examples/parse_timestamp_tagged_logs.rs
index ca11dcd..d201380 100644
--- a/examples/parse_timestamp_tagged_logs.rs
+++ b/examples/parse_timestamp_tagged_logs.rs
@@ -3,64 +3,154 @@ use std::fs::File;
 use std::io;
 use std::io::BufRead;
 use std::path::{Path, PathBuf};
+use std::time::Duration;
 use regex::Regex;
+use clap::Parser;
+
+pub fn parse_log_entry_subscriber(log_entry: &str) -> (u64, u64) {
 
-pub fn parse_log_entry_subscriber(log_entry: &str) -> HashMap<String, String> {
     let re = Regex::new(r".*got account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
     let caps = re.captures(log_entry).unwrap();
 
-    let mut result = HashMap::new();
-    result.insert("write_version".to_string(), caps["write_version"].to_string());
-    result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
-    result.insert("slot".to_string(), caps["slot"].to_string());
+    // let mut result = HashMap::new();
+    // result.insert("write_version".to_string(), caps["write_version"].to_string());
+    // result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
+    // result.insert("slot".to_string(), caps["slot"].to_string());
+
+    let write_version: u64 = caps["write_version"].parse().unwrap();
+    let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
 
-    result
+    (write_version, timestamp_us)
 }
 
-pub fn parse_log_entry_source(log_entry: &str) -> HashMap<String, String> {
+pub fn parse_log_entry_source(log_entry: &str) -> (u64, u64) {
     let re = Regex::new(r".*account update: write_version=(?P<write_version>\d+);timestamp_us=(?P<timestamp_us>\d+);slot=(?P<slot>\d+)").unwrap();
     let caps = re.captures(log_entry).unwrap();
 
-    let mut result = HashMap::new();
-    result.insert("write_version".to_string(), caps["write_version"].to_string());
-    result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
-    result.insert("slot".to_string(), caps["slot"].to_string());
+    // let mut result = HashMap::new();
+    // result.insert("write_version".to_string(), caps["write_version"].to_string());
+    // result.insert("timestamp_us".to_string(), caps["timestamp_us"].to_string());
+    // result.insert("slot".to_string(), caps["slot"].to_string());
+
+    let write_version: u64 = caps["write_version"].parse().unwrap();
+    let timestamp_us: u64 = caps["timestamp_us"].parse().unwrap();
 
-    result
+    (write_version, timestamp_us)
 }
 
 
 
-fn read_subscriber_log(log_file: PathBuf) {
+fn read_subscriber_log(log_file: PathBuf) -> HashMap<u64, u64> {
+    let mut map: HashMap<u64, u64> = HashMap::new();
 
     let file = File::open(log_file).expect("file must exist");
     let reader = io::BufReader::new(file);
-    for line in reader.lines().take(10) {
+    for line in reader.lines().take(1000) {
         let line = line.expect("must be parsable");
-        let parsed = parse_log_entry_subscriber(&line);
-        println!("{:?}", parsed);
+        let (write_version, timestamp_us) = parse_log_entry_subscriber(&line);
+        // println!("{:?}", parsed);
+        map.insert(write_version, timestamp_us);
     }
 
+    map
 }
 
-fn read_source_log(log_file: PathBuf) {
+fn read_source_log(log_file: PathBuf) -> HashMap<u64, u64> {
+    let mut map: HashMap<u64, u64> = HashMap::new();
+
     let file = File::open(log_file).expect("file must exist");
     let reader = io::BufReader::new(file);
-    for line in reader.lines().take(10) {
+    for line in reader.lines().take(1000) {
         let line = line.expect("must be parsable");
-        let parsed = parse_log_entry_source(&line);
-        println!("{:?}", parsed);
+        let (write_version, timestamp_us) = parse_log_entry_source(&line);
+        // println!("{:?}", parsed);
+        map.insert(write_version, timestamp_us);
     }
+
+    map
+}
+
+// cat macbook.log |cut -b 111- | tr -d 'a-z_=' > macbook.log.csv
+// cat solana-validator-macbook.log | cut -b 96- | tr -d 'a-z_='
+fn read_from_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
+    csv::ReaderBuilder::new()
+        .delimiter(b';')
+        .has_headers(false)
+        .from_path(csv_file)
+        .unwrap()
+        .into_deserialize()
+        .map(|record| {
+            let record: Vec<String> = record.unwrap();
+            let write_version = record[0].parse::<u64>().unwrap();
+            let timestamp_us = record[1].parse::<u64>().unwrap();
+            (write_version, timestamp_us)
+        })
+        .collect::<HashMap<u64, u64>>()
+}
+
+fn read_subscriber_log_csv(csv_file: PathBuf) -> HashMap<u64, u64> {
+    csv::ReaderBuilder::new()
+        .delimiter(b';')
+        .has_headers(false)
+        .from_path(csv_file)
+        .unwrap()
+        .into_deserialize()
+        .map(|record| {
+            let record: Vec<String> = record.unwrap();
+            let write_version = record[0].parse::<u64>().unwrap();
+            let timestamp_us = record[1].parse::<u64>().unwrap();
+            (write_version, timestamp_us)
+        })
+        .collect::<HashMap<u64, u64>>()
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+pub struct Args {
+    #[arg(long)]
+    pub csv_file_source: String,
+    #[arg(long)]
+    pub csv_file_subscriber: String,
 }
 
 
 pub fn main() {
+    let Args { csv_file_source, csv_file_subscriber } = Args::parse();
 
+    println!("Reading source log ...");
+    let source_timestamps = read_from_csv(PathBuf::from(csv_file_source));
 
-    read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
+    println!("Reading subscriber log ...");
+    let subscriber_timestamps = read_from_csv(PathBuf::from(csv_file_subscriber));
 
-    read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
+    for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
+        if let Some(source_timestamp) = source_timestamps.get(&write_version) {
+            let diff = (timestamp_us as i128) - (*source_timestamp as i128);
+            println!("write_version: {}, subscriber: {}, source: {}, diff: {:.1}ms", write_version, timestamp_us, source_timestamp, diff as f64 / 1000.0);
+        }
 
+    }
+
+}
+
+pub fn main__() {
+    println!("Reading subscriber log ...");
+    let subscriber_timestamps = read_subscriber_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/macbook.log"));
+
+    println!("Reading source log ...");
+    let source_timestamps = read_source_log(PathBuf::from("/Users/stefan/mango/projects/geyser-misc/accounts-stream-performance/solana-validator-macbook.log"));
+
+    println!("Comparing ...");
+
+    for (write_version, timestamp_us) in subscriber_timestamps.into_iter() {
+
+        // println!("write_version: {}, subscriber: {}", write_version, timestamp_us);
+        if let Some(source_timestamp) = source_timestamps.get(&write_version) {
+            let diff = (timestamp_us as i128) - (*source_timestamp as i128);
+            println!("write_version: {}, subscriber: {}, source: {}, diff: {}", write_version, timestamp_us, source_timestamp, diff);
+        }
+
+    }
 
 
 }

From 087625de3fe2a639e3aa9470faea6ca08456aed3 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 7 May 2024 17:34:58 +0200
Subject: [PATCH 35/67] track phoenix orderbook

---
 examples/stream_token_accounts.rs | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index 531e753..b5ca82f 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -71,7 +71,8 @@ pub async fn main() {
                         Some(UpdateOneof::Account(update)) => {
                             let account = update.account.unwrap();
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
-                            trace!("got account update (green)!!! {} - {:?} - {} bytes",
+                            let size = account.data.len();
+                            info!("got account update (green)!!! {} - {:?} - {} bytes",
                                 update.slot, account_pk, account.data.len());
 
                             if ENABLE_TIMESTAMP_TAGGING {
@@ -79,6 +80,7 @@ pub async fn main() {
                                 info!("got account update: write_version={};timestamp_us={};slot={}", account.write_version, since_the_epoch.as_micros(), update.slot);
                             }
 
+
                             match parse_token(&account.data, Some(6)) {
                                 Ok(TokenAccountType::Account(account_ui)) => {
                                     // UiTokenAccount {
@@ -110,6 +112,10 @@ pub async fn main() {
                                     if account_ui.owner.starts_with("66fEFnKy") {
                                         info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
                                     }
+                                    // if pubkey.starts_with(b"JUP") {
+                                    //     info!("update balance for mint {} of owner {}: {}", mint, owner, amount);
+                                    // }
+
                                     token_account_by_ownermint.entry(owner)
                                         .or_insert_with(DashMap::new)
                                         .insert(mint, account_ui);
@@ -167,9 +173,9 @@ pub fn token_accounts() -> SubscribeRequest {
     accounts_subs.insert(
         "client".to_string(),
         SubscribeRequestFilterAccounts {
-            account: vec![],
-            owner:
-                spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
+            account: vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            owner: vec![],
+                // spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
         },
     );

From 88f8d71ba8008f4db43d9969c4a63294d14fb4fb Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 7 May 2024 20:31:19 +0200
Subject: [PATCH 36/67] count data

---
 examples/stream_token_accounts.rs | 50 ++++++++++++++++++++++++++-----
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index b5ca82f..0af0d03 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -19,11 +19,11 @@ use tokio::time::{sleep, Duration};
 use tracing::field::debug;
 use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
-use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeUpdate};
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
 
 
-const ENABLE_TIMESTAMP_TAGGING: bool = true;
+const ENABLE_TIMESTAMP_TAGGING: bool = false;
 
 #[tokio::main]
 pub async fn main() {
@@ -63,16 +63,27 @@ pub async fn main() {
 
 
     tokio::spawn(async move {
+        let mut bytes_per_slot: HashMap<Slot, u64> = HashMap::new();
+        let mut updates_per_slot: HashMap<Slot, u64> = HashMap::new();
+
+        let mut changing_slot = 0;
+        let mut current_slot = 0;
+
+
         let mut green_stream = pin!(green_stream);
         while let Some(message) = green_stream.next().await {
             match message {
                 Message::GeyserSubscribeUpdate(subscriber_update) => {
                     match subscriber_update.update_oneof {
+                        Some(UpdateOneof::Slot(update)) => {
+                            current_slot = update.slot;
+                        }
                         Some(UpdateOneof::Account(update)) => {
+                            let slot = update.slot as Slot;
                             let account = update.account.unwrap();
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
-                            let size = account.data.len();
-                            info!("got account update (green)!!! {} - {:?} - {} bytes",
+                            let size = account.data.len() as u64;
+                            trace!("got account update (green)!!! {} - {:?} - {} bytes",
                                 update.slot, account_pk, account.data.len());
 
                             if ENABLE_TIMESTAMP_TAGGING {
@@ -120,6 +131,22 @@ pub async fn main() {
                                         .or_insert_with(DashMap::new)
                                         .insert(mint, account_ui);
 
+                                    bytes_per_slot.entry(slot)
+                                        .and_modify(|total| *total += size).or_insert(size);
+
+                                    updates_per_slot.entry(slot)
+                                        .and_modify(|total| *total += 1).or_insert(1);
+
+                                    info!("delta: {}", (slot as i64) - (current_slot as i64));
+
+                                    if slot != changing_slot && changing_slot != 0 {
+                                        let total_bytes = bytes_per_slot.get(&changing_slot).unwrap();
+                                        let updates_count = updates_per_slot.get(&changing_slot).unwrap();
+                                        // info!("Slot {} - Total bytes: {} in {} updates", slot, total_bytes, updates_count);
+                                    }
+                                    changing_slot = slot;
+
+
                                 }
                                 Ok(TokenAccountType::Mint(mint)) => {
                                     // not interesting
@@ -173,16 +200,23 @@ pub fn token_accounts() -> SubscribeRequest {
     accounts_subs.insert(
         "client".to_string(),
         SubscribeRequestFilterAccounts {
-            account: vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
-            owner: vec![],
-                // spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
+            account: vec![],
+            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            owner:
+                spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
         },
     );
 
 
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
+        filter_by_commitment: Some(true),
+    });
+
+
     SubscribeRequest {
-        slots: HashMap::new(),
+        slots: slots_subs,
         accounts: accounts_subs,
         transactions: HashMap::new(),
         entry: Default::default(),

From 193a1b6878fd6bbf970d8c85ef0b5062de63daf5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 11:20:58 +0200
Subject: [PATCH 37/67] make SubscribeRequest forward-compatible

---
 src/lib.rs | 32 ++++----------------------------
 1 file changed, 4 insertions(+), 28 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 1ec5715..4a5b434 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -100,15 +100,9 @@ impl GeyserFilter {
         );
 
         SubscribeRequest {
-            slots: HashMap::new(),
-            accounts: Default::default(),
-            transactions: HashMap::new(),
-            entry: Default::default(),
             blocks: blocks_subs,
-            blocks_meta: HashMap::new(),
             commitment: Some(map_commitment_level(self.0) as i32),
-            accounts_data_slice: Default::default(),
-            ping: None,
+            ..Default::default()
         }
     }
 
@@ -117,15 +111,9 @@ impl GeyserFilter {
         blocksmeta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {});
 
         SubscribeRequest {
-            slots: HashMap::new(),
-            accounts: Default::default(),
-            transactions: HashMap::new(),
-            entry: Default::default(),
-            blocks: HashMap::new(),
             blocks_meta: blocksmeta_subs,
             commitment: Some(map_commitment_level(self.0) as i32),
-            accounts_data_slice: Default::default(),
-            ping: None,
+            ..Default::default()
         }
     }
 
@@ -140,14 +128,8 @@ impl GeyserFilter {
 
         SubscribeRequest {
             slots: slots_subs,
-            accounts: Default::default(),
-            transactions: HashMap::new(),
-            entry: Default::default(),
-            blocks: HashMap::new(),
-            blocks_meta: HashMap::new(),
             commitment: Some(map_commitment_level(self.0) as i32),
-            accounts_data_slice: Default::default(),
-            ping: None,
+            ..Default::default()
         }
     }
 
@@ -163,15 +145,9 @@ impl GeyserFilter {
         );
 
         SubscribeRequest {
-            slots: HashMap::new(),
             accounts: accounts_subs,
-            transactions: HashMap::new(),
-            entry: Default::default(),
-            blocks: Default::default(),
-            blocks_meta: HashMap::new(),
             commitment: Some(map_commitment_level(self.0) as i32),
-            accounts_data_slice: Default::default(),
-            ping: None,
+            ..Default::default()
         }
     }
 }

From 78b81976683b1313795b0759bd2b8c4a581b6cd2 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 15:25:06 +0200
Subject: [PATCH 38/67] calc delta using two slot sources

---
 Cargo.lock                             |   2 +
 Cargo.toml                             |   1 +
 examples/bench_geyser_grpc_accounts.rs | 241 ++++++++++++++++++++-----
 3 files changed, 201 insertions(+), 43 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e760222..e906d14 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1378,6 +1378,7 @@ dependencies = [
  "solana-logger",
  "solana-sdk",
  "tokio",
+ "tonic",
  "tonic-health",
  "tracing",
  "tracing-subscriber",
@@ -3759,6 +3760,7 @@ dependencies = [
  "axum",
  "base64 0.21.7",
  "bytes",
+ "flate2",
  "h2",
  "http",
  "http-body",
diff --git a/Cargo.toml b/Cargo.toml
index ddce63b..177d7cd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -33,6 +33,7 @@ csv = "1.3.0"
 
 dashmap = "5.5.3"
 
+tonic = { version="0.10.2", features=["gzip"] }
 tonic-health = "0.10.2"
 regex = "1.10.4"
 clap = { version = "4.2", features = ["derive"] }
diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 1556f83..bdda15d 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -1,23 +1,26 @@
 use std::collections::{HashMap, VecDeque};
 use futures::{Stream, StreamExt};
-use log::info;
+use log::{debug, info};
 use solana_sdk::clock::{Slot, UnixTimestamp};
 use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::pubkey::Pubkey;
 use std::env;
 use std::pin::pin;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 use itertools::Itertools;
+use solana_account_decoder::parse_token::spl_token_ids;
 use tokio::sync::mpsc::Receiver;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
 use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
-use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
-use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, histogram_percentiles, Message};
+use geyser_grpc_connector::grpcmultiplex_fastestwins::{create_multiplexed_stream, FromYellowstoneExtractor};
+use geyser_grpc_connector::{AtomicSlot, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, histogram_percentiles, Message};
 use tokio::time::{sleep, Duration};
-use tracing::warn;
+use tracing::{trace, warn};
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
-use yellowstone_grpc_proto::geyser::SubscribeUpdate;
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
 
 #[tokio::main]
@@ -49,30 +52,115 @@ pub async fn main() {
     let (autoconnect_tx, geyser_messages_rx) = tokio::sync::mpsc::channel(10);
     let (_exit, exit_notify) = tokio::sync::broadcast::channel(1);
 
-    let _accounts_task = create_geyser_autoconnection_task_with_mpsc(
-        config.clone(),
-        GeyserFilter(CommitmentConfig::processed()).accounts(),
-        autoconnect_tx.clone(),
-        exit_notify.resubscribe(),
-    );
+    // let _accounts_task = create_geyser_autoconnection_task_with_mpsc(
+    //     config.clone(),
+    //     GeyserFilter(CommitmentConfig::processed()).accounts(),
+    //     autoconnect_tx.clone(),
+    //     exit_notify.resubscribe(),
+    // );
+    //
+    // let _blocksmeta_task = create_geyser_autoconnection_task_with_mpsc(
+    //     config.clone(),
+    //     GeyserFilter(CommitmentConfig::processed()).blocks_meta(),
+    //     autoconnect_tx.clone(),
+    //     exit_notify.resubscribe(),
+    // );
 
-    let _blocksmeta_task = create_geyser_autoconnection_task_with_mpsc(
+    let _all_accounts_and_blocksmeta_task = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
-        GeyserFilter(CommitmentConfig::processed()).blocks_meta(),
+        all_accounts_and_blocksmeta(),
         autoconnect_tx.clone(),
         exit_notify.resubscribe(),
     );
 
-    start_tracking_account_consumer(geyser_messages_rx);
+    // let _token_accounts_task = create_geyser_autoconnection_task_with_mpsc(
+    //     config.clone(),
+    //     token_accounts(),
+    //     autoconnect_tx.clone(),
+    //     exit_notify.resubscribe(),
+    // );
+
+    let current_processed_slot = AtomicSlot::default();
+    start_tracking_slots(current_processed_slot.clone());
+    start_tracking_account_consumer(geyser_messages_rx, current_processed_slot.clone());
 
     // "infinite" sleep
     sleep(Duration::from_secs(1800)).await;
 }
 
-// note: this keeps track of lot of data and might blow up memory
-fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
-    const RECENT_SLOTS_LIMIT: usize = 30;
+
+
+// note processed might return a slot that night end up on a fork
+fn start_tracking_slots(current_processed_slot: AtomicSlot) {
+
+    let grpc_slot_source1 = env::var("GRPC_SLOT1_ADDR").expect("need grpc url for green");
+    let grpc_slot_source2 = env::var("GRPC_SLOT2_ADDR").expect("need grpc url for green");
+
+    info!("Using grpc sources for slot: {}, {}",
+        grpc_slot_source1, grpc_slot_source2
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(5),
+        request_timeout: Duration::from_secs(5),
+        subscribe_timeout: Duration::from_secs(5),
+        receive_timeout: Duration::from_secs(5),
+    };
+
+    let config1 = GrpcSourceConfig::new(grpc_slot_source1, None, None, timeouts.clone());
+    let config2 = GrpcSourceConfig::new(grpc_slot_source2, None, None, timeouts.clone());
+
 
+    tokio::spawn(async move {
+        debug!("start tracking slots..");
+
+        let (multiplex_tx, mut multiplex_rx) = tokio::sync::mpsc::channel(10);
+        // TODO expose
+        let (_exit, exit_notify) = tokio::sync::broadcast::channel(1);
+
+        let _blocksmeta_task1 = create_geyser_autoconnection_task_with_mpsc(
+            config1.clone(),
+            GeyserFilter(CommitmentConfig::processed()).slots(),
+            multiplex_tx.clone(),
+            exit_notify.resubscribe(),
+        );
+
+        let _blocksmeta_task2 = create_geyser_autoconnection_task_with_mpsc(
+            config2.clone(),
+            GeyserFilter(CommitmentConfig::processed()).slots(),
+            multiplex_tx.clone(),
+            exit_notify.resubscribe(),
+        );
+
+        let mut tip: Slot = 0;
+
+        loop {
+            match multiplex_rx.recv().await {
+                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
+                    Some(UpdateOneof::Slot(update)) => {
+                        let slot = update.slot;
+                        if slot > tip {
+                            tip = slot;
+                            current_processed_slot.store(slot, Ordering::Relaxed);
+                        }
+                    }
+                    None => {}
+                    _ => {}
+                },
+                None => {
+                    log::warn!("multiplexer channel closed - aborting");
+                    return;
+                }
+                Some(Message::Connecting(_)) => {}
+            }
+        }
+
+    });
+
+}
+
+// note: this keeps track of lot of data and might blow up memory
+fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, current_processed_slot: Arc<AtomicU64>) {
     tokio::spawn(async move {
 
         let mut bytes_per_slot = HashMap::<Slot, usize>::new();
@@ -80,14 +168,11 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
         let mut wallclock_updates_per_slot_account = HashMap::<(Slot, Pubkey), Vec<SystemTime>>::new();
         // slot written by account update
         let mut current_slot: Slot = 0;
-        // slot from slot stream
-        let mut slot_just_completed: Slot = 0;
 
         // seconds since epoch
         let mut block_time_per_slot = HashMap::<Slot, UnixTimestamp>::new();
         // wall clock time of block completion (i.e. processed) reported by the block meta stream
         let mut block_completion_notification_time_per_slot = HashMap::<Slot, SystemTime>::new();
-        let mut recent_slot_deltas: VecDeque<i64> = VecDeque::with_capacity(RECENT_SLOTS_LIMIT);
 
         loop {
             match geyser_messages_rx.recv().await {
@@ -100,16 +185,19 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                         let slot = update.slot;
                         let account_receive_time = get_epoch_sec();
 
-                        if slot_just_completed != slot {
-                            if slot_just_completed != 0 {
-                                // the perfect is value "-1"
-                                recent_slot_deltas.push_back((slot_just_completed as i64) - (slot as i64));
-                                if recent_slot_deltas.len() > RECENT_SLOTS_LIMIT {
-                                    recent_slot_deltas.pop_front();
-                                }
-                            }
+                        let latest_slot = current_processed_slot.load(Ordering::Relaxed);
+
+                        if latest_slot != 0 {
+                            // the perfect is value "-1"
+                            let delta = (latest_slot as i64) - (slot as i64);
+                            debug!("Account info for upcoming slot {} was {} behind current processed slot", slot, delta);
                         }
 
+                        // if account_info.data.len() > 1000 {
+                        //     trace!("got account update!!! {} - {:?} - {} bytes",
+                        //         slot, account_pk, account_info.data.len());
+                        // }
+
                         bytes_per_slot.entry(slot)
                             .and_modify(|entry| *entry += account_info.data.len())
                             .or_insert(account_info.data.len());
@@ -135,14 +223,6 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                                 let count_histogram = histogram_percentiles::calculate_percentiles(&counters);
                                 info!("Count histogram: {}", count_histogram);
 
-                                let deltas = recent_slot_deltas.iter()
-                                    .map(|x| *x as f64)
-                                    .sorted_by(|a, b| a.partial_cmp(b).unwrap())
-                                    .collect_vec();
-                                let deltas_histogram = histogram_percentiles::calculate_percentiles(&deltas);
-                                info!("Deltas slots list: {:?}", recent_slot_deltas);
-                                info!("Deltas histogram: {}", deltas_histogram);
-
                                 if let Some(actual_block_time) = block_time_per_slot.get(&current_slot) {
                                     info!("Block time for slot {}: delta {} seconds", current_slot, account_receive_time - *actual_block_time);
                                 }
@@ -164,13 +244,6 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>) {
                         }
 
                     }
-                    Some(UpdateOneof::BlockMeta(update)) => {
-                        let now = SystemTime::now();
-                        // completed depends on commitment level which is processed ATM
-                        slot_just_completed = update.slot;
-                        block_time_per_slot.insert(slot_just_completed, update.block_time.unwrap().timestamp);
-                        block_completion_notification_time_per_slot.insert(slot_just_completed, now);
-                    }
                     None => {}
                     _ => {}
                 },
@@ -190,3 +263,85 @@ fn get_epoch_sec() -> UnixTimestamp {
         .unwrap()
         .as_secs() as UnixTimestamp
 }
+
+
+pub fn token_accounts() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            owner:
+            spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
+            filters: vec![],
+        },
+    );
+
+
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
+        filter_by_commitment: Some(true),
+    });
+
+    let mut blocks_meta_subs = HashMap::new();
+    blocks_meta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {});
+
+    SubscribeRequest {
+        slots: slots_subs,
+        accounts: accounts_subs,
+        blocks_meta: blocks_meta_subs,
+        ..Default::default()
+    }
+}
+
+
+pub fn all_accounts_and_blocksmeta() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![],
+            filters: vec![],
+        },
+    );
+
+
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
+        filter_by_commitment: Some(true),
+    });
+
+    let mut blocks_meta_subs = HashMap::new();
+    blocks_meta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {});
+
+    SubscribeRequest {
+        slots: slots_subs,
+        accounts: accounts_subs,
+        blocks_meta: blocks_meta_subs,
+        ..Default::default()
+    }
+}
+
+
+pub fn all_accounts() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![],
+            filters: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        accounts: accounts_subs,
+        ..Default::default()
+    }
+}
+
+
+
+

From 86412cfd71ff82712a969cdacf394dfe9db64d25 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 15:25:32 +0200
Subject: [PATCH 39/67] define AtomicSlot

---
 src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lib.rs b/src/lib.rs
index b35f513..61a2e09 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,6 +1,8 @@
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::collections::HashMap;
 use std::fmt::{Debug, Display};
+use std::sync::Arc;
+use std::sync::atomic::AtomicU64;
 use std::time::Duration;
 use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterBlocks, SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::transport::ClientTlsConfig;
@@ -15,6 +17,8 @@ mod obfuscate;
 pub mod histogram_percentiles;
 pub mod yellowstone_grpc_util;
 
+pub type AtomicSlot = Arc<AtomicU64>;
+
 // 1-based attempt counter
 type Attempt = u32;
 

From 430847b2c124e7b44dbab2a6688f7354e3da1419 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 15:53:16 +0200
Subject: [PATCH 40/67] debounce update log

---
 examples/bench_geyser_grpc_accounts.rs   |  8 +++++-
 examples/debouncer.rs                    | 35 ++++++++++++++++++++++++
 examples/stream_blocks_mainnet_stream.rs |  6 ++--
 3 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 examples/debouncer.rs

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index bdda15d..d323539 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -23,6 +23,8 @@ use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
 
+mod debouncer;
+
 #[tokio::main]
 pub async fn main() {
     // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
@@ -174,6 +176,8 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
         // wall clock time of block completion (i.e. processed) reported by the block meta stream
         let mut block_completion_notification_time_per_slot = HashMap::<Slot, SystemTime>::new();
 
+        let debouncer = debouncer::Debouncer::new(Duration::from_millis(5));
+
         loop {
             match geyser_messages_rx.recv().await {
                 Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
@@ -190,7 +194,9 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                         if latest_slot != 0 {
                             // the perfect is value "-1"
                             let delta = (latest_slot as i64) - (slot as i64);
-                            debug!("Account info for upcoming slot {} was {} behind current processed slot", slot, delta);
+                            if debouncer.can_fire() {
+                                debug!("Account info for upcoming slot {} was {} behind current processed slot", slot, delta);
+                            }
                         }
 
                         // if account_info.data.len() > 1000 {
diff --git a/examples/debouncer.rs b/examples/debouncer.rs
new file mode 100644
index 0000000..49ef1ad
--- /dev/null
+++ b/examples/debouncer.rs
@@ -0,0 +1,35 @@
+use std::sync::atomic::{AtomicI64, Ordering};
+use std::time::{Duration, Instant};
+
+#[derive(Debug)]
+pub struct Debouncer {
+    started_at: Instant,
+    cooldown_ms: i64,
+    last: AtomicI64,
+}
+
+impl Debouncer {
+    pub fn new(cooldown: Duration) -> Self {
+        Self {
+            started_at: Instant::now(),
+            cooldown_ms: cooldown.as_millis() as i64,
+            last: AtomicI64::new(0),
+        }
+    }
+    pub fn can_fire(&self) -> bool {
+        let passed_total_ms = self.started_at.elapsed().as_millis() as i64;
+
+        let results = self.last.fetch_update(Ordering::SeqCst, Ordering::SeqCst,
+             |last| {
+
+                 if passed_total_ms - last > self.cooldown_ms {
+                     Some(passed_total_ms)
+                 } else {
+                     None
+                 }
+
+        });
+
+        results.is_ok()
+    }
+}
diff --git a/examples/stream_blocks_mainnet_stream.rs b/examples/stream_blocks_mainnet_stream.rs
index e2c2731..302da98 100644
--- a/examples/stream_blocks_mainnet_stream.rs
+++ b/examples/stream_blocks_mainnet_stream.rs
@@ -14,7 +14,7 @@ use solana_sdk::compute_budget::ComputeBudgetInstruction;
 use solana_sdk::hash::Hash;
 use solana_sdk::instruction::CompiledInstruction;
 use solana_sdk::message::v0::MessageAddressTableLookup;
-use solana_sdk::message::{v0, MessageHeader, VersionedMessage};
+use solana_sdk::message::{MessageHeader, v0, VersionedMessage};
 use solana_sdk::pubkey::Pubkey;
 
 use solana_sdk::signature::Signature;
@@ -26,10 +26,12 @@ use geyser_grpc_connector::grpcmultiplex_fastestwins::{
     create_multiplexed_stream, FromYellowstoneExtractor,
 };
 use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig};
-use tokio::time::{sleep, Duration};
+use tokio::time::{Duration, sleep};
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::SubscribeUpdate;
 
+pub mod debouncer;
+
 fn start_example_block_consumer(
     multiplex_stream: impl Stream<Item = ProducedBlock> + Send + 'static,
 ) {

From 61b202af9f5a2bd9bc4fc7089e0bebd67a1cac52 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 16:14:22 +0200
Subject: [PATCH 41/67] stop subscribing blocks meta

---
 examples/bench_geyser_grpc_accounts.rs | 66 +++++++++++++-------------
 1 file changed, 32 insertions(+), 34 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index d323539..1fa277b 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -8,7 +8,7 @@ use std::env;
 use std::pin::pin;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use itertools::Itertools;
 use solana_account_decoder::parse_token::spl_token_ids;
 use tokio::sync::mpsc::Receiver;
@@ -68,9 +68,9 @@ pub async fn main() {
     //     exit_notify.resubscribe(),
     // );
 
-    let _all_accounts_and_blocksmeta_task = create_geyser_autoconnection_task_with_mpsc(
+    let _all_accounts = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
-        all_accounts_and_blocksmeta(),
+        all_accounts(),
         autoconnect_tx.clone(),
         exit_notify.resubscribe(),
     );
@@ -182,6 +182,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
             match geyser_messages_rx.recv().await {
                 Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
                     Some(UpdateOneof::Account(update)) => {
+                        let started_at = Instant::now();
                         let now = SystemTime::now();
                         let account_info = update.account.unwrap();
                         let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
@@ -214,40 +215,37 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                             .and_modify(|entry| entry.push(now))
                             .or_insert(vec![now]);
 
-                        if current_slot != slot {
-                            info!("Slot: {}", slot);
-                            if current_slot != 0 {
-                                info!("Slot: {} - account data transferred: {:.2} MiB", slot, *bytes_per_slot.get(&current_slot).unwrap() as f64 / 1024.0 / 1024.0 );
-
-                                info!("Slot: {} - num of update messages: {}", slot, updates_per_slot.get(&current_slot).unwrap());
-
-                                let counters = wallclock_updates_per_slot_account.iter()
-                                    .filter(|((slot, _pubkey), _)| slot == &current_slot)
-                                    .map(|((_slot, _pubkey), updates)| updates.len() as f64)
-                                    .sorted_by(|a, b| a.partial_cmp(b).unwrap())
-                                    .collect_vec();
-                                let count_histogram = histogram_percentiles::calculate_percentiles(&counters);
-                                info!("Count histogram: {}", count_histogram);
-
-                                if let Some(actual_block_time) = block_time_per_slot.get(&current_slot) {
-                                    info!("Block time for slot {}: delta {} seconds", current_slot, account_receive_time - *actual_block_time);
-                                }
-
-                                let wallclock_minmax = wallclock_updates_per_slot_account.iter()
-                                    .filter(|((slot, _pubkey), _)| slot == &current_slot)
-                                    .flat_map(|((_slot, _pubkey), updates)| updates)
-                                    .minmax();
-                                if let Some((min, max)) = wallclock_minmax.into_option() {
-                                    info!("Wallclock timestamp between first and last account update received for slot {}: {:.2}s",
-                                        current_slot,
-                                        max.duration_since(*min).unwrap().as_secs_f64()
-                                    );
-                                }
+                        if current_slot != slot && current_slot != 0 {
+                            info!("New Slot: {}", slot);
+                            info!("Slot: {} - account data transferred: {:.2} MiB", slot, *bytes_per_slot.get(&current_slot).unwrap() as f64 / 1024.0 / 1024.0 );
 
+                            info!("Slot: {} - num of update messages: {}", slot, updates_per_slot.get(&current_slot).unwrap());
 
+                            let counters = wallclock_updates_per_slot_account.iter()
+                                .filter(|((slot, _pubkey), _)| slot == &current_slot)
+                                .map(|((_slot, _pubkey), updates)| updates.len() as f64)
+                                .sorted_by(|a, b| a.partial_cmp(b).unwrap())
+                                .collect_vec();
+                            let count_histogram = histogram_percentiles::calculate_percentiles(&counters);
+                            info!("Count histogram: {}", count_histogram);
+
+                            if let Some(actual_block_time) = block_time_per_slot.get(&current_slot) {
+                                info!("Block time for slot {}: delta {} seconds", current_slot, account_receive_time - *actual_block_time);
                             }
-                            current_slot = slot;
-                        }
+
+                            let wallclock_minmax = wallclock_updates_per_slot_account.iter()
+                                .filter(|((slot, _pubkey), _)| slot == &current_slot)
+                                .flat_map(|((_slot, _pubkey), updates)| updates)
+                                .minmax();
+                            if let Some((min, max)) = wallclock_minmax.into_option() {
+                                info!("Wallclock timestamp between first and last account update received for slot {}: {:.2}s",
+                                    current_slot,
+                                    max.duration_since(*min).unwrap().as_secs_f64()
+                                );
+                            }
+
+                        } // -- slot changed
+                        current_slot = slot;
 
                     }
                     None => {}

From 6ffc8d0f0d95260b1deaae054a1f9ea3ce72aa73 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 16:15:20 +0200
Subject: [PATCH 42/67] bench ONLY spl accounts

---
 examples/bench_geyser_grpc_accounts.rs | 29 ++++++++------------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 1fa277b..d3fa11a 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -68,20 +68,20 @@ pub async fn main() {
     //     exit_notify.resubscribe(),
     // );
 
-    let _all_accounts = create_geyser_autoconnection_task_with_mpsc(
-        config.clone(),
-        all_accounts(),
-        autoconnect_tx.clone(),
-        exit_notify.resubscribe(),
-    );
-
-    // let _token_accounts_task = create_geyser_autoconnection_task_with_mpsc(
+    // let _all_accounts = create_geyser_autoconnection_task_with_mpsc(
     //     config.clone(),
-    //     token_accounts(),
+    //     all_accounts(),
     //     autoconnect_tx.clone(),
     //     exit_notify.resubscribe(),
     // );
 
+    let _token_accounts_task = create_geyser_autoconnection_task_with_mpsc(
+        config.clone(),
+        token_accounts(),
+        autoconnect_tx.clone(),
+        exit_notify.resubscribe(),
+    );
+
     let current_processed_slot = AtomicSlot::default();
     start_tracking_slots(current_processed_slot.clone());
     start_tracking_account_consumer(geyser_messages_rx, current_processed_slot.clone());
@@ -282,19 +282,8 @@ pub fn token_accounts() -> SubscribeRequest {
         },
     );
 
-
-    let mut slots_subs = HashMap::new();
-    slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
-        filter_by_commitment: Some(true),
-    });
-
-    let mut blocks_meta_subs = HashMap::new();
-    blocks_meta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {});
-
     SubscribeRequest {
-        slots: slots_subs,
         accounts: accounts_subs,
-        blocks_meta: blocks_meta_subs,
         ..Default::default()
     }
 }

From 32aeb332b14e6287d8461ab8eb92c33c0cf29660 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 8 May 2024 16:45:13 +0200
Subject: [PATCH 43/67] try to enable compression

---
 src/yellowstone_grpc_util.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/yellowstone_grpc_util.rs b/src/yellowstone_grpc_util.rs
index c2831a9..534ce6d 100644
--- a/src/yellowstone_grpc_util.rs
+++ b/src/yellowstone_grpc_util.rs
@@ -1,4 +1,5 @@
 use std::time::Duration;
+use tonic::codec::CompressionEncoding;
 use tonic_health::pb::health_client::HealthClient;
 use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientResult, InterceptorXToken};
 use yellowstone_grpc_proto::geyser::geyser_client::GeyserClient;
@@ -110,6 +111,7 @@ where
     let client = GeyserGrpcClient::new(
         HealthClient::with_interceptor(channel.clone(), interceptor.clone()),
         GeyserClient::with_interceptor(channel, interceptor)
+            .accept_compressed(CompressionEncoding::Gzip)
             .max_decoding_message_size(GeyserGrpcClient::max_decoding_message_size()),
     );
     Ok(client)

From f6206641b97363591836fc016a91a868f18108ab Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 10 May 2024 18:21:57 +0200
Subject: [PATCH 44/67] fix ordering of slot change

---
 examples/bench_geyser_grpc_accounts.rs | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index d3fa11a..1f0dda3 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -190,15 +190,6 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                         let slot = update.slot;
                         let account_receive_time = get_epoch_sec();
 
-                        let latest_slot = current_processed_slot.load(Ordering::Relaxed);
-
-                        if latest_slot != 0 {
-                            // the perfect is value "-1"
-                            let delta = (latest_slot as i64) - (slot as i64);
-                            if debouncer.can_fire() {
-                                debug!("Account info for upcoming slot {} was {} behind current processed slot", slot, delta);
-                            }
-                        }
 
                         // if account_info.data.len() > 1000 {
                         //     trace!("got account update!!! {} - {:?} - {} bytes",
@@ -221,13 +212,13 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
 
                             info!("Slot: {} - num of update messages: {}", slot, updates_per_slot.get(&current_slot).unwrap());
 
-                            let counters = wallclock_updates_per_slot_account.iter()
+                            let per_account_updates = wallclock_updates_per_slot_account.iter()
                                 .filter(|((slot, _pubkey), _)| slot == &current_slot)
                                 .map(|((_slot, _pubkey), updates)| updates.len() as f64)
                                 .sorted_by(|a, b| a.partial_cmp(b).unwrap())
                                 .collect_vec();
-                            let count_histogram = histogram_percentiles::calculate_percentiles(&counters);
-                            info!("Count histogram: {}", count_histogram);
+                            let per_account_updates_histogram = histogram_percentiles::calculate_percentiles(&per_account_updates);
+                            info!("Per-account updates histogram: {}", per_account_updates_histogram);
 
                             if let Some(actual_block_time) = block_time_per_slot.get(&current_slot) {
                                 info!("Block time for slot {}: delta {} seconds", current_slot, account_receive_time - *actual_block_time);
@@ -247,6 +238,17 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                         } // -- slot changed
                         current_slot = slot;
 
+
+                        let latest_slot = current_processed_slot.load(Ordering::Relaxed);
+
+                        if latest_slot != 0 {
+                            // the perfect is value "-1"
+                            let delta = (latest_slot as i64) - (slot as i64);
+                            if debouncer.can_fire() {
+                                debug!("Account info for upcoming slot {} was {} behind current processed slot", slot, delta);
+                            }
+                        }
+
                     }
                     None => {}
                     _ => {}

From 4157779f12dc004a80363f1990c249f7955f8e96 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 10 May 2024 18:23:16 +0200
Subject: [PATCH 45/67] subscribe to all accounts

---
 examples/bench_geyser_grpc_accounts.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 1f0dda3..bcac977 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -68,20 +68,20 @@ pub async fn main() {
     //     exit_notify.resubscribe(),
     // );
 
-    // let _all_accounts = create_geyser_autoconnection_task_with_mpsc(
-    //     config.clone(),
-    //     all_accounts(),
-    //     autoconnect_tx.clone(),
-    //     exit_notify.resubscribe(),
-    // );
-
-    let _token_accounts_task = create_geyser_autoconnection_task_with_mpsc(
+    let _all_accounts = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
-        token_accounts(),
+        all_accounts(),
         autoconnect_tx.clone(),
         exit_notify.resubscribe(),
     );
 
+    // let _token_accounts_task = create_geyser_autoconnection_task_with_mpsc(
+    //     config.clone(),
+    //     token_accounts(),
+    //     autoconnect_tx.clone(),
+    //     exit_notify.resubscribe(),
+    // );
+
     let current_processed_slot = AtomicSlot::default();
     start_tracking_slots(current_processed_slot.clone());
     start_tracking_account_consumer(geyser_messages_rx, current_processed_slot.clone());

From 03b3d140311143d2312bc196f88b73102ade85c4 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 13 May 2024 11:12:31 +0200
Subject: [PATCH 46/67] add x-token for slot1+slot2

---
 examples/bench_geyser_grpc_accounts.rs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index bcac977..ab91175 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -95,8 +95,11 @@ pub async fn main() {
 // note processed might return a slot that night end up on a fork
 fn start_tracking_slots(current_processed_slot: AtomicSlot) {
 
-    let grpc_slot_source1 = env::var("GRPC_SLOT1_ADDR").expect("need grpc url for green");
-    let grpc_slot_source2 = env::var("GRPC_SLOT2_ADDR").expect("need grpc url for green");
+    let grpc_slot_source1 = env::var("GRPC_SLOT1_ADDR").expect("need grpc url for slot source1");
+    let grpc_x_token_source1 = env::var("GRPC_SLOT1_X_TOKEN").ok();
+
+    let grpc_slot_source2 = env::var("GRPC_SLOT2_ADDR").expect("need grpc url for slot source2");
+    let grpc_x_token_source2 = env::var("GRPC_SLOT2_X_TOKEN").ok();
 
     info!("Using grpc sources for slot: {}, {}",
         grpc_slot_source1, grpc_slot_source2
@@ -109,8 +112,8 @@ fn start_tracking_slots(current_processed_slot: AtomicSlot) {
         receive_timeout: Duration::from_secs(5),
     };
 
-    let config1 = GrpcSourceConfig::new(grpc_slot_source1, None, None, timeouts.clone());
-    let config2 = GrpcSourceConfig::new(grpc_slot_source2, None, None, timeouts.clone());
+    let config1 = GrpcSourceConfig::new(grpc_slot_source1, grpc_x_token_source1, None, timeouts.clone());
+    let config2 = GrpcSourceConfig::new(grpc_slot_source2, grpc_x_token_source2, None, timeouts.clone());
 
 
     tokio::spawn(async move {

From 08f17d34b17dc9431220d091f0529c772004aacd Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 13 May 2024 11:38:24 +0200
Subject: [PATCH 47/67] account to CSV

---
 examples/accounts_meta_to_csv.rs | 43 ++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 examples/accounts_meta_to_csv.rs

diff --git a/examples/accounts_meta_to_csv.rs b/examples/accounts_meta_to_csv.rs
new file mode 100644
index 0000000..70abbb2
--- /dev/null
+++ b/examples/accounts_meta_to_csv.rs
@@ -0,0 +1,43 @@
+use std::fs::File;
+use std::io;
+use std::io::BufRead;
+use std::path::PathBuf;
+use itertools::Itertools;
+
+pub fn main() {
+
+    let accounts_meta_file = PathBuf::from("/Users/stefan/mango/projects/geyser-misc/ledger-debug-accounts.txt");
+
+    let file = File::open(accounts_meta_file).expect("file must exist");
+    let reader = io::BufReader::new(file);
+    for blocks in &reader.lines().chunks(9) {
+        let blocks = blocks.collect_vec();
+        let account_pk = blocks[0].as_ref().unwrap().replace(":", "");
+        if account_pk == "" {
+            break;
+        }
+        let owner_pk = blocks[2].as_ref().unwrap();
+        let ltick = owner_pk.find("'");
+        let rtick = owner_pk.rfind("'");
+        let owner_pk = &owner_pk[ltick.unwrap() + 1..rtick.unwrap()];
+
+        let data_len = blocks[6].as_ref().unwrap().replace("  data_len: ", "");
+
+        println!("{};{};{}", account_pk, owner_pk, data_len);
+
+    }
+
+}
+
+
+/*
+16FMCmgLzCNNz6eTwGanbyN2ZxvTBSLuQ6DZhgeMshg:
+  balance: 0.00095352 SOL
+  owner: 'Feature111111111111111111111111111111111111'
+  executable: false
+  slot: 0
+  rent_epoch: 0
+  data_len: 9
+  data: 'AQAAAAAAAAAA'
+  encoding: "base64"
+*/

From 71d49a58ac06d56f5e7a2905639d9ee3ccddf619 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 14 May 2024 14:18:35 +0200
Subject: [PATCH 48/67] token accounts logging

---
 examples/stream_token_accounts.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index 0af0d03..efdea13 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -83,7 +83,7 @@ pub async fn main() {
                             let account = update.account.unwrap();
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
                             let size = account.data.len() as u64;
-                            trace!("got account update (green)!!! {} - {:?} - {} bytes",
+                            trace!("got account update: {} - {:?} - {} bytes",
                                 update.slot, account_pk, account.data.len());
 
                             if ENABLE_TIMESTAMP_TAGGING {
@@ -137,12 +137,15 @@ pub async fn main() {
                                     updates_per_slot.entry(slot)
                                         .and_modify(|total| *total += 1).or_insert(1);
 
-                                    info!("delta: {}", (slot as i64) - (current_slot as i64));
+                                    let delta = (slot as i64) - (current_slot as i64);
+                                    if delta > 1 {
+                                        info!("delta: {}", (slot as i64) - (current_slot as i64));
+                                    }
 
                                     if slot != changing_slot && changing_slot != 0 {
                                         let total_bytes = bytes_per_slot.get(&changing_slot).unwrap();
                                         let updates_count = updates_per_slot.get(&changing_slot).unwrap();
-                                        // info!("Slot {} - Total bytes: {} in {} updates", slot, total_bytes, updates_count);
+                                        info!("Slot {} - Total bytes: {}, {} updates", slot, total_bytes, updates_count);
                                     }
                                     changing_slot = slot;
 

From 57104022c5e1f6570bd682050c9960e4edb83c8a Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 17 May 2024 16:41:06 +0200
Subject: [PATCH 49/67] parse for time tagged log files

---
 examples/parse_yellowstone_timetagged_log.rs | 159 +++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 examples/parse_yellowstone_timetagged_log.rs

diff --git a/examples/parse_yellowstone_timetagged_log.rs b/examples/parse_yellowstone_timetagged_log.rs
new file mode 100644
index 0000000..1a8183c
--- /dev/null
+++ b/examples/parse_yellowstone_timetagged_log.rs
@@ -0,0 +1,159 @@
+use std::collections::HashMap;
+use std::fs::File;
+use std::io;
+use std::io::BufRead;
+use std::path::PathBuf;
+use clap::Parser;
+use regex::Regex;
+
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+pub struct Args {
+    #[arg(long)]
+    pub log_file: String,
+}
+
+
+pub fn main() {
+    let Args { log_file } = Args::parse();
+    println!("Reading log file: {}", log_file);
+
+    let log_file = PathBuf::from(log_file);
+
+    const LIMIT_LINES: usize = 10000000;
+
+    let mut timetag_sending_to_buffer: HashMap<u64, u64> = HashMap::new();
+    let mut timetag_before_sending_grpc: HashMap<u64, u64> = HashMap::new();
+    // contains only matches from previous sets
+    let mut timetag_geyser: HashMap<u64, u64> = HashMap::new();
+
+    let mut count_sending_to_buffer_channel = 0;
+    {
+        let file = File::open(&log_file).expect("file must exist");
+        let reader = io::BufReader::new(file);
+        for line in reader.lines().take(LIMIT_LINES) {
+            let line = line.expect("must be parsable");
+            // println!("-> buffer channel");
+            if let Some((write_version, timestamp_us)) = parse_log_entry_sending_to_buffer_channel(line) {
+                count_sending_to_buffer_channel += 1;
+                timetag_sending_to_buffer.insert(write_version, timestamp_us);
+            }
+        }
+    }
+
+    let mut count_sending_grpc = 0;
+    {
+        let file = File::open(&log_file).expect("file must exist");
+        let reader = io::BufReader::new(file);
+        for line in reader.lines().take(LIMIT_LINES) {
+            let line = line.expect("must be parsable");
+            // println!("-> when sending to grpc");
+            if let Some((write_version, timestamp_us)) = parse_log_entry_before_sending_grpc(line) {
+                count_sending_grpc += 1;
+                timetag_before_sending_grpc.insert(write_version, timestamp_us);
+            }
+        }
+    }
+
+    // THIS is by far the largest set 
+    let mut count_at_geyser = 0;
+    {
+        let file = File::open(&log_file).expect("file must exist");
+        let reader = io::BufReader::new(file);
+        for line in reader.lines().take(LIMIT_LINES) {
+            let line = line.expect("must be parsable");
+            // println!("-> at geyser interface");
+            if let Some((write_version, timestamp_us)) = parse_log_entry_at_geyser_interface(line) {
+                count_at_geyser += 1;
+                if timetag_sending_to_buffer.contains_key(&write_version) && timetag_before_sending_grpc.contains_key(&write_version) {
+                    timetag_geyser.insert(write_version, timestamp_us);
+                }
+            }
+        }
+    }
+
+    println!("Count at geyser interface: {}", count_at_geyser);
+    println!("Count sending to buffer channel: {}", count_sending_to_buffer_channel);
+    println!("Count sending to grpc: {}", count_sending_grpc);
+
+    for (write_version, geyser_timestamp_us) in timetag_geyser {
+        let timestamp_sending_to_buffer = timetag_sending_to_buffer.get(&write_version).unwrap();
+        let timestamp_before_sending_grpc = timetag_before_sending_grpc.get(&write_version).unwrap();
+        let delta1 = timestamp_sending_to_buffer - geyser_timestamp_us;
+        let delta2 = timestamp_before_sending_grpc - timestamp_sending_to_buffer;
+        println!("Write Version: {}, geyser - {}us - buffer - {}us - grpc", write_version, delta1, delta2);
+
+    }
+
+}
+
+
+fn parse_log_entry_at_geyser_interface(log_line: String) -> Option<(u64, u64)> {
+    if !log_line.contains("account update inspect from geyser") {
+        return None;
+    }
+
+    // Split the log line by ': ' to separate the prefix from the data
+    let parts: Vec<&str> = log_line.split(": ").collect();
+
+// The second part contains the data we need
+    let data = parts[1];
+
+// Split the data by ';' to separate the different fields
+    let fields: Vec<&str> = data.split(';').collect();
+
+// For each field, split by '=' to separate the key from the value
+    let write_version: u64 = fields[0].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+    let timestamp_us: u64 = fields[1].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+    let _slot: u64 = fields[2].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+
+    Some((write_version, timestamp_us))
+}
+
+
+fn parse_log_entry_sending_to_buffer_channel(log_line: String) -> Option<(u64, u64)> {
+    if !log_line.contains("sending to buffer channel") {
+        return None;
+    }
+
+// Split the log line by ': ' to separate the prefix from the data
+    let parts: Vec<&str> = log_line.split(": ").collect();
+
+// The second part contains the data we need
+    let data = parts[1];
+
+// Split the data by ';' to separate the different fields
+    let fields: Vec<&str> = data.split(';').collect();
+
+// For each field, split by '=' to separate the key from the value
+    let write_version: u64 = fields[0].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+    let timestamp_us: u64 = fields[1].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+    let _slot: u64 = fields[2].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+
+    Some((write_version, timestamp_us))
+}
+
+
+fn parse_log_entry_before_sending_grpc(log_line: String) -> Option<(u64, u64)> {
+    if !log_line.contains("before sending to grpc") {
+        return None;
+    }
+
+// Split the log line by ': ' to separate the prefix from the data
+    let parts: Vec<&str> = log_line.split(": ").collect();
+
+// The third part contains the data we need
+    let data = parts[1];
+
+// Split the data by ';' to separate the different fields
+    let fields: Vec<&str> = data.split(';').collect();
+
+// For each field, split by '=' to separate the key from the value
+    let write_version: u64 = fields[0].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+    let timestamp_us: u64 = fields[1].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+    let _slot: u64 = fields[2].split('=').collect::<Vec<&str>>()[1].parse().unwrap();
+
+    Some((write_version, timestamp_us))
+}
+

From e366b59dbbdf53635acd7a60a2ee3dc75c8f46fd Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 6 Jun 2024 10:51:00 +0200
Subject: [PATCH 50/67] read only vote transactions

---
 examples/stream_vote_transactions.rs | 134 +++++++++++++++++++++++++++
 1 file changed, 134 insertions(+)
 create mode 100644 examples/stream_vote_transactions.rs

diff --git a/examples/stream_vote_transactions.rs b/examples/stream_vote_transactions.rs
new file mode 100644
index 0000000..688792f
--- /dev/null
+++ b/examples/stream_vote_transactions.rs
@@ -0,0 +1,134 @@
+use std::collections::HashMap;
+use futures::{Stream, StreamExt};
+use log::{debug, info, trace};
+use solana_sdk::clock::Slot;
+use solana_sdk::commitment_config::CommitmentConfig;
+use std::env;
+use std::pin::pin;
+use std::str::FromStr;
+use std::sync::Arc;
+use dashmap::DashMap;
+use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType, UiTokenAccount};
+use solana_account_decoder::parse_token::UiAccountState::Initialized;
+use solana_sdk::program_utils::limited_deserialize;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::vote::instruction::VoteInstruction;
+use solana_sdk::vote::state::Vote;
+
+use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
+use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
+use tokio::time::{sleep, Duration};
+use tracing::field::debug;
+use tracing::warn;
+use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, SubscribeUpdate};
+use yellowstone_grpc_proto::prost::Message as _;
+
+
+const ENABLE_TIMESTAMP_TAGGING: bool = false;
+
+#[tokio::main]
+pub async fn main() {
+    // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
+    tracing_subscriber::fmt::init();
+    // console_subscriber::init();
+
+    let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green");
+    let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok();
+
+    info!(
+        "Using grpc source on {} ({})",
+        grpc_addr_green,
+        grpc_x_token_green.is_some()
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(25),
+        request_timeout: Duration::from_secs(25),
+        subscribe_timeout: Duration::from_secs(25),
+        receive_timeout: Duration::from_secs(25),
+    };
+
+    let config = GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
+
+    info!("Write Block stream..");
+
+    let green_stream = create_geyser_reconnecting_stream(
+        config.clone(),
+        transaction_filter(),
+    );
+
+    tokio::spawn(async move {
+        let mut green_stream = pin!(green_stream);
+        while let Some(message) = green_stream.next().await {
+            match message {
+                Message::GeyserSubscribeUpdate(subscriber_update) => {
+                    match subscriber_update.update_oneof {
+                        Some(UpdateOneof::Transaction(update)) => {
+
+                            let message = update.transaction.unwrap().transaction.unwrap().message.unwrap();
+
+
+                            // https://docs.solanalabs.com/implemented-proposals/validator-timestamp-oracle
+                            for ci in message.instructions {
+                                let vote_instruction = limited_deserialize::<VoteInstruction>(&ci.data).unwrap();
+                                info!("vote_instruction: {:?}", vote_instruction);
+                                info!("vote_instruction: {:?}", vote_instruction.timestamp().unwrap());
+                            }
+
+                            // let is_vote_transaction = message.instructions().iter().any(|i| {
+                            //     i.program_id(message.static_account_keys())
+                            //         .eq(&solana_sdk::vote::program::id())
+                            //         && limited_deserialize::<VoteInstruction>(&i.data)
+                            //         .map(|vi| vi.is_simple_vote())
+                            //         .unwrap_or(false)
+                            // });
+
+
+
+                        }
+                        _ => {}
+                    }
+                }
+                Message::Connecting(attempt) => {
+                    warn!("Connection attempt: {}", attempt);
+                }
+            }
+        }
+        warn!("Stream aborted");
+    });
+
+
+    // "infinite" sleep
+    sleep(Duration::from_secs(1800)).await;
+}
+
+pub fn transaction_filter() -> SubscribeRequest {
+    let mut trnasactions_subs = HashMap::new();
+    trnasactions_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterTransactions {
+            vote: Some(true),
+            failed: Some(false),
+            signature: None,
+            // TODO
+            account_include: vec![],
+            account_exclude: vec![],
+            account_required: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        slots: HashMap::new(),
+        accounts: HashMap::new(),
+        transactions: trnasactions_subs,
+        entry: Default::default(),
+        blocks: Default::default(),
+        blocks_meta: HashMap::new(),
+        commitment: None,
+        accounts_data_slice: Default::default(),
+        ping: None,
+    }
+}
+

From a57860ec14f32b91f5bfebaa542c94c17ff764b5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 6 Jun 2024 11:06:56 +0200
Subject: [PATCH 51/67] calculate vote timestamp spread

---
 examples/stream_vote_transactions.rs | 33 +++++++++++++++++++---------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/examples/stream_vote_transactions.rs b/examples/stream_vote_transactions.rs
index 688792f..462ac4a 100644
--- a/examples/stream_vote_transactions.rs
+++ b/examples/stream_vote_transactions.rs
@@ -1,13 +1,14 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use futures::{Stream, StreamExt};
 use log::{debug, info, trace};
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Slot, UnixTimestamp};
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
 use std::pin::pin;
 use std::str::FromStr;
 use std::sync::Arc;
 use dashmap::DashMap;
+use itertools::Itertools;
 use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType, UiTokenAccount};
 use solana_account_decoder::parse_token::UiAccountState::Initialized;
 use solana_sdk::program_utils::limited_deserialize;
@@ -24,6 +25,7 @@ use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
+use geyser_grpc_connector::histogram_percentiles::calculate_percentiles;
 
 
 const ENABLE_TIMESTAMP_TAGGING: bool = false;
@@ -60,6 +62,9 @@ pub async fn main() {
     );
 
     tokio::spawn(async move {
+
+        let mut vote_times_by_slot: HashMap<Slot, HashSet<UnixTimestamp>> = HashMap::new();
+
         let mut green_stream = pin!(green_stream);
         while let Some(message) = green_stream.next().await {
             match message {
@@ -68,23 +73,23 @@ pub async fn main() {
                         Some(UpdateOneof::Transaction(update)) => {
 
                             let message = update.transaction.unwrap().transaction.unwrap().message.unwrap();
+                            let slot = update.slot;
 
 
                             // https://docs.solanalabs.com/implemented-proposals/validator-timestamp-oracle
                             for ci in message.instructions {
                                 let vote_instruction = limited_deserialize::<VoteInstruction>(&ci.data).unwrap();
-                                info!("vote_instruction: {:?}", vote_instruction);
+                                let last_voted_slot = vote_instruction.last_voted_slot().unwrap();
                                 info!("vote_instruction: {:?}", vote_instruction.timestamp().unwrap());
+                                vote_times_by_slot.entry(last_voted_slot).or_insert(HashSet::new()).insert(vote_instruction.timestamp().unwrap());
                             }
 
-                            // let is_vote_transaction = message.instructions().iter().any(|i| {
-                            //     i.program_id(message.static_account_keys())
-                            //         .eq(&solana_sdk::vote::program::id())
-                            //         && limited_deserialize::<VoteInstruction>(&i.data)
-                            //         .map(|vi| vi.is_simple_vote())
-                            //         .unwrap_or(false)
-                            // });
 
+                            // hack to look at reasonable settled slot
+                            // print_spread(&vote_times_by_slot, slot);
+                            if vote_times_by_slot.contains_key(&(slot-10)) {
+                                print_spread(&vote_times_by_slot, slot - 10);
+                            }
 
 
                         }
@@ -104,6 +109,14 @@ pub async fn main() {
     sleep(Duration::from_secs(1800)).await;
 }
 
+fn print_spread(vote_times_by_slot: &HashMap<Slot, HashSet<UnixTimestamp>>, slot: Slot) {
+    let slots = vote_times_by_slot.get(&slot).unwrap();
+    let min_slot = slots.iter().min().unwrap();
+    let array = slots.iter().sorted().map(|x| (*x - min_slot) as f64).collect_vec();
+    let histo = calculate_percentiles(&array);
+    info!("slot: {} histo: {}", slot, histo);
+}
+
 pub fn transaction_filter() -> SubscribeRequest {
     let mut trnasactions_subs = HashMap::new();
     trnasactions_subs.insert(

From beeca856bc091d3737317f0944e8f30a28fabafb Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 7 Jun 2024 10:40:13 +0200
Subject: [PATCH 52/67] fix account bench

---
 examples/bench_geyser_grpc_accounts.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index ab91175..168ef98 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -83,7 +83,7 @@ pub async fn main() {
     // );
 
     let current_processed_slot = AtomicSlot::default();
-    start_tracking_slots(current_processed_slot.clone());
+    // start_tracking_slots(current_processed_slot.clone());
     start_tracking_account_consumer(geyser_messages_rx, current_processed_slot.clone());
 
     // "infinite" sleep
@@ -179,7 +179,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
         // wall clock time of block completion (i.e. processed) reported by the block meta stream
         let mut block_completion_notification_time_per_slot = HashMap::<Slot, SystemTime>::new();
 
-        let debouncer = debouncer::Debouncer::new(Duration::from_millis(5));
+        let debouncer = debouncer::Debouncer::new(Duration::from_millis(50));
 
         loop {
             match geyser_messages_rx.recv().await {

From 759170c66052f89fdee7ad9e9957f387b3a6c2e9 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 7 Jun 2024 13:57:05 +0200
Subject: [PATCH 53/67] collect hashes

---
 examples/bench_geyser_grpc_accounts.rs | 42 ++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 168ef98..3b88d78 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -6,11 +6,13 @@ use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::pubkey::Pubkey;
 use std::env;
 use std::pin::pin;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use itertools::Itertools;
 use solana_account_decoder::parse_token::spl_token_ids;
+use solana_sdk::hash::{Hash, hash};
 use tokio::sync::mpsc::Receiver;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
@@ -173,6 +175,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
         let mut wallclock_updates_per_slot_account = HashMap::<(Slot, Pubkey), Vec<SystemTime>>::new();
         // slot written by account update
         let mut current_slot: Slot = 0;
+        let mut account_hashes = HashMap::<Pubkey, Vec<Hash>>::new();
 
         // seconds since epoch
         let mut block_time_per_slot = HashMap::<Slot, UnixTimestamp>::new();
@@ -181,6 +184,11 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
 
         let debouncer = debouncer::Debouncer::new(Duration::from_millis(50));
 
+        // Phoenix
+        let selected_account_pk = Pubkey::from_str("4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg").unwrap();
+        let mut last_account_data: Option<Vec<u8>> = None;
+
+
         loop {
             match geyser_messages_rx.recv().await {
                 Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
@@ -189,16 +197,44 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                         let now = SystemTime::now();
                         let account_info = update.account.unwrap();
                         let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
+                        let account_owner_pk = Pubkey::try_from(account_info.owner).unwrap();
                         // note: slot is referencing the block that is just built while the slot number reported from BlockMeta/Slot uses the slot after the block is built
                         let slot = update.slot;
                         let account_receive_time = get_epoch_sec();
 
+                        if account_info.data.len() > 100000 {
+                            let hash = hash(&account_info.data);
+                            // info!("got account update!!! {} - {:?} - {} bytes - {} - {}lamps",
+                            //     slot, account_pk, account_info.data.len(), hash, account_info.lamports);
 
-                        // if account_info.data.len() > 1000 {
-                        //     trace!("got account update!!! {} - {:?} - {} bytes",
-                        //         slot, account_pk, account_info.data.len());
+                            account_hashes.entry(account_pk)
+                                .and_modify(|entry| entry.push(hash))
+                                .or_insert(vec![hash]);
+                        }
+
+                        // if account_hashes.len() > 100 {
+                        //     for (pubkey, hashes) in &account_hashes {
+                        //         info!("account hashes for {:?}", pubkey);
+                        //         for hash in hashes {
+                        //             info!("- hash: {}", hash);
+                        //         }
+                        //     }
                         // }
 
+                        if account_pk == selected_account_pk {
+                            info!("got account update!!! {} - {:?} - {} bytes - {}",
+                                slot, account_pk, account_info.data.len(), account_info.lamports);
+
+                            if let Some(data) = last_account_data {
+                                let hash1 = hash(&data);
+                                let hash2 = hash(&account_info.data);
+                                info!("diff: {} {}", hash1, hash2);
+                            }
+
+                            last_account_data = Some(account_info.data.clone());
+                        }
+
+
                         bytes_per_slot.entry(slot)
                             .and_modify(|entry| *entry += account_info.data.len())
                             .or_insert(account_info.data.len());

From af17800bba8d8fd39817b6ad7bbe804cf3b7ceba Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 7 Jun 2024 14:11:12 +0200
Subject: [PATCH 54/67] xor+lz4

---
 Cargo.lock                             | 26 +++++++++++++++++++++
 Cargo.toml                             |  2 ++
 examples/bench_geyser_grpc_accounts.rs | 32 ++++++++++++++++++++++++--
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e906d14..13a96b5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1372,6 +1372,7 @@ dependencies = [
  "futures",
  "itertools 0.10.5",
  "log",
+ "lz4_flex",
  "merge-streams",
  "regex",
  "solana-account-decoder",
@@ -1846,6 +1847,15 @@ version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
 
+[[package]]
+name = "lz4_flex"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
+dependencies = [
+ "twox-hash",
+]
+
 [[package]]
 name = "matchit"
 version = "0.7.3"
@@ -3493,6 +3503,12 @@ dependencies = [
  "spl-program-error",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -3902,6 +3918,16 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
+[[package]]
+name = "twox-hash"
+version = "1.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
+dependencies = [
+ "cfg-if",
+ "static_assertions",
+]
+
 [[package]]
 name = "typenum"
 version = "1.17.0"
diff --git a/Cargo.toml b/Cargo.toml
index 177d7cd..f5699ea 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -38,6 +38,8 @@ tonic-health = "0.10.2"
 regex = "1.10.4"
 clap = { version = "4.2", features = ["derive"] }
 
+lz4_flex = "0.11.3"
+
 [dev-dependencies]
 tracing-subscriber = "0.3.16"
 solana-logger = "1"
diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 3b88d78..c82e2d5 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -1,3 +1,4 @@
+use std::cmp::min;
 use std::collections::{HashMap, VecDeque};
 use futures::{Stream, StreamExt};
 use log::{debug, info};
@@ -225,10 +226,15 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                             info!("got account update!!! {} - {:?} - {} bytes - {}",
                                 slot, account_pk, account_info.data.len(), account_info.lamports);
 
-                            if let Some(data) = last_account_data {
-                                let hash1 = hash(&data);
+                            if let Some(prev_data) = last_account_data {
+                                let hash1 = hash(&prev_data);
                                 let hash2 = hash(&account_info.data);
                                 info!("diff: {} {}", hash1, hash2);
+                                
+                                if hash1 != hash2 {
+                                    delta_compress(&prev_data, &account_info.data);
+                                }
+                                
                             }
 
                             last_account_data = Some(account_info.data.clone());
@@ -302,6 +308,28 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
     });
 }
 
+fn delta_compress(prev_data: &Vec<u8>, data: &Vec<u8>) {
+
+    let xor_region = min(prev_data.len(), data.len());
+    let mut xor_diff = vec![0u8; xor_region];
+
+    for i in 0..xor_region {
+        xor_diff[i] = prev_data[i] ^ data[i];
+    }
+
+    // TODO https://users.rust-lang.org/t/how-to-find-common-prefix-of-two-byte-slices-effectively/25815/3
+    let count_non_zero = xor_diff.iter().filter(|&x| *x != 0).count();
+    info!("count_non_zero={} xor_region={}", count_non_zero, xor_region);
+    // info!("hex {:02X?}", xor_data);
+
+    let compressed_xor = lz4_flex::compress_prepend_size(&xor_diff);
+    info!("compressed size of xor: {} (was {})", compressed_xor.len(), xor_diff.len());
+
+    let compressed_data = lz4_flex::compress_prepend_size(&data);
+    info!("compressed size of data: {} (was {})", compressed_data.len(), data.len());
+
+}
+
 fn get_epoch_sec() -> UnixTimestamp {
     SystemTime::now()
         .duration_since(UNIX_EPOCH)

From 460ca4c56fa319673dc9ceb0e7d6245d61dcf1ce Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Wed, 12 Jun 2024 18:07:26 +0200
Subject: [PATCH 55/67] data delta

---
 examples/bench_geyser_grpc_accounts.rs | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index c82e2d5..5c50c38 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -5,7 +5,7 @@ use log::{debug, info};
 use solana_sdk::clock::{Slot, UnixTimestamp};
 use solana_sdk::commitment_config::CommitmentConfig;
 use solana_sdk::pubkey::Pubkey;
-use std::env;
+use std::{env, iter};
 use std::pin::pin;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -185,8 +185,10 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
 
         let debouncer = debouncer::Debouncer::new(Duration::from_millis(50));
 
-        // Phoenix
-        let selected_account_pk = Pubkey::from_str("4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg").unwrap();
+        // Phoenix 4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg
+        // CzK26LWpoU9UjSrZkVu97oZj63abJrNv1zp9Hy2zZdy5
+        // 6ojSigXF7nDPyhFRgmn3V9ywhYseKF9J32ZrranMGVSX
+        // FV8EEHjJvDUD8Kkp1DcomTatZBA81Z6C5AhmvyUwvEAh
         let mut last_account_data: Option<Vec<u8>> = None;
 
 
@@ -231,10 +233,8 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                                 let hash2 = hash(&account_info.data);
                                 info!("diff: {} {}", hash1, hash2);
                                 
-                                if hash1 != hash2 {
-                                    delta_compress(&prev_data, &account_info.data);
-                                }
-                                
+                                delta_compress(&prev_data, &account_info.data);
+
                             }
 
                             last_account_data = Some(account_info.data.clone());
@@ -313,11 +313,18 @@ fn delta_compress(prev_data: &Vec<u8>, data: &Vec<u8>) {
     let xor_region = min(prev_data.len(), data.len());
     let mut xor_diff = vec![0u8; xor_region];
 
+    let mut equal = 0;
     for i in 0..xor_region {
         xor_diff[i] = prev_data[i] ^ data[i];
+        equal |= xor_diff[i];
     }
 
-    // TODO https://users.rust-lang.org/t/how-to-find-common-prefix-of-two-byte-slices-effectively/25815/3
+    if equal == 0 && prev_data.len() == data.len() {
+        info!("no difference in data");
+        return;
+    }
+
+
     let count_non_zero = xor_diff.iter().filter(|&x| *x != 0).count();
     info!("count_non_zero={} xor_region={}", count_non_zero, xor_region);
     // info!("hex {:02X?}", xor_data);

From 238f44ac52d5e35c423b2602e0baa461715992cf Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 13 Jun 2024 15:34:13 +0200
Subject: [PATCH 56/67] fix compile

---
 examples/bench_geyser_grpc_accounts.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 5c50c38..333a283 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -189,6 +189,9 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
         // CzK26LWpoU9UjSrZkVu97oZj63abJrNv1zp9Hy2zZdy5
         // 6ojSigXF7nDPyhFRgmn3V9ywhYseKF9J32ZrranMGVSX
         // FV8EEHjJvDUD8Kkp1DcomTatZBA81Z6C5AhmvyUwvEAh
+        // choose an account for which the diff should be calculated
+        let selected_account_pk = Pubkey::from_str("4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg").unwrap();
+
         let mut last_account_data: Option<Vec<u8>> = None;
 
 
@@ -240,7 +243,6 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                             last_account_data = Some(account_info.data.clone());
                         }
 
-
                         bytes_per_slot.entry(slot)
                             .and_modify(|entry| *entry += account_info.data.len())
                             .or_insert(account_info.data.len());

From 3fd7e6d9405d427fcfd5d500f45613d4d256c289 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 13 Jun 2024 15:42:48 +0200
Subject: [PATCH 57/67] enable slot check

---
 examples/bench_geyser_grpc_accounts.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 333a283..d097f7e 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -86,7 +86,7 @@ pub async fn main() {
     // );
 
     let current_processed_slot = AtomicSlot::default();
-    // start_tracking_slots(current_processed_slot.clone());
+    start_tracking_slots(current_processed_slot.clone());
     start_tracking_account_consumer(geyser_messages_rx, current_processed_slot.clone());
 
     // "infinite" sleep
@@ -292,7 +292,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                             // the perfect is value "-1"
                             let delta = (latest_slot as i64) - (slot as i64);
                             if debouncer.can_fire() {
-                                debug!("Account info for upcoming slot {} was {} behind current processed slot", slot, delta);
+                                debug!("Account info for upcoming slot {} was {} behind current processed slot (best is -1)", slot, delta);
                             }
                         }
 

From 63196123eb5446d47d293d655f57c18ee7c3d4d7 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 13 Jun 2024 15:57:04 +0200
Subject: [PATCH 58/67] smarter lag logging

---
 examples/bench_geyser_grpc_accounts.rs | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index d097f7e..16cbe29 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -292,7 +292,19 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                             // the perfect is value "-1"
                             let delta = (latest_slot as i64) - (slot as i64);
                             if debouncer.can_fire() {
-                                debug!("Account info for upcoming slot {} was {} behind current processed slot (best is -1)", slot, delta);
+                                let is_lagging = delta > -1;
+                                let is_lagging_a_lot = delta - 20 > -1;
+                                let info_text = if is_lagging {
+                                    if is_lagging_a_lot {
+                                        "A LOT"
+                                    } else {
+                                        "a bit"
+                                    }
+                                } else {
+                                    "good"
+                                };
+                                // Account info for upcoming slot {} was {} behind current processed slot
+                                debug!("Update slot {}, delta: {} - {}", slot, delta, info_text);
                             }
                         }
 

From a4a93dea8f82826cf968e9a51fabef283b0fbde5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 13 Jun 2024 16:06:33 +0200
Subject: [PATCH 59/67] allow for mock reset

---
 examples/bench_geyser_grpc_accounts.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/examples/bench_geyser_grpc_accounts.rs b/examples/bench_geyser_grpc_accounts.rs
index 16cbe29..e717dc2 100644
--- a/examples/bench_geyser_grpc_accounts.rs
+++ b/examples/bench_geyser_grpc_accounts.rs
@@ -147,10 +147,13 @@ fn start_tracking_slots(current_processed_slot: AtomicSlot) {
                 Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
                     Some(UpdateOneof::Slot(update)) => {
                         let slot = update.slot;
-                        if slot > tip {
-                            tip = slot;
-                            current_processed_slot.store(slot, Ordering::Relaxed);
-                        }
+                        current_processed_slot.store(slot, Ordering::Relaxed);
+
+                        // don't do that with the mock impl as the slots restart when mock restarts
+                        // if slot > tip {
+                        //     tip = slot;
+                        //     current_processed_slot.store(slot, Ordering::Relaxed);
+                        // }
                     }
                     None => {}
                     _ => {}
@@ -285,7 +288,6 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                         } // -- slot changed
                         current_slot = slot;
 
-
                         let latest_slot = current_processed_slot.load(Ordering::Relaxed);
 
                         if latest_slot != 0 {
@@ -304,7 +306,7 @@ fn start_tracking_account_consumer(mut geyser_messages_rx: Receiver<Message>, cu
                                     "good"
                                 };
                                 // Account info for upcoming slot {} was {} behind current processed slot
-                                debug!("Update slot {}, delta: {} - {}", slot, delta, info_text);
+                                debug!("Account update slot {}, delta: {} - {}", slot, delta, info_text);
                             }
                         }
 

From c096a04125eeb9111e46c10f30153c59686a85a7 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 14 Jun 2024 10:16:16 +0200
Subject: [PATCH 60/67] set nodelay + add env for window+biffer sizes

---
 src/grpc_subscription_autoreconnect_tasks.rs | 23 ++++++++++++++++++--
 src/yellowstone_grpc_util.rs                 |  2 ++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index f15741c..1ea0150 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -1,3 +1,4 @@
+use std::env;
 use std::future::Future;
 use crate::{yellowstone_grpc_util, Attempt, GrpcSourceConfig, Message};
 use futures::{Stream, StreamExt};
@@ -13,7 +14,7 @@ use yellowstone_grpc_client::{GeyserGrpcClient, GeyserGrpcClientError};
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeUpdate};
 use yellowstone_grpc_proto::tonic::service::Interceptor;
 use yellowstone_grpc_proto::tonic::Status;
-use crate::yellowstone_grpc_util::connect_with_timeout_with_buffers;
+use crate::yellowstone_grpc_util::{connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig};
 
 enum ConnectionState<S: Stream<Item = Result<SubscribeUpdate, Status>>, F: Interceptor> {
     NotConnected(Attempt),
@@ -88,7 +89,8 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                         addr
                     );
 
-                    let buffer_config = yellowstone_grpc_util::GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter);
+                    // let buffer_config = yellowstone_grpc_util::GeyserGrpcClientBufferConfig::optimize_for_subscription(&subscribe_filter);
+                    let buffer_config = buffer_config_from_env();
                     debug!("Using Grpc Buffer config {:?}", buffer_config);
 
                     let connection_handler = |connect_result| match connect_result {
@@ -374,6 +376,23 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
     jh_geyser_task
 }
 
+fn buffer_config_from_env() -> GeyserGrpcClientBufferConfig {
+    if env::var("BUFFER_SIZE").is_err() || env::var("CONN_WINDOW").is_err() || env::var("STREAM_WINDOW").is_err() {
+        return GeyserGrpcClientBufferConfig::default();
+    }
+
+    let buffer_size = env::var("BUFFER_SIZE").expect("buffer_size").parse::<usize>().expect("integer(bytes)");
+    let conn_window = env::var("CONN_WINDOW").expect("conn_window").parse::<u32>().expect("integer(bytes)");
+    let stream_window = env::var("STREAM_WINDOW").expect("stream_window").parse::<u32>().expect("integer(bytes)");
+
+    // conn_window should be larger than stream_window
+    GeyserGrpcClientBufferConfig {
+        buffer_size: Some(buffer_size),
+        conn_window: Some(conn_window),
+        stream_window: Some(stream_window),
+    }
+}
+
 
 enum MaybeExit<T> {
     Continue(T),
diff --git a/src/yellowstone_grpc_util.rs b/src/yellowstone_grpc_util.rs
index 534ce6d..f160e4f 100644
--- a/src/yellowstone_grpc_util.rs
+++ b/src/yellowstone_grpc_util.rs
@@ -85,6 +85,8 @@ where
 {
     // see https://github.com/blockworks-foundation/geyser-grpc-connector/issues/10
     let mut endpoint = tonic::transport::Endpoint::from_shared(endpoint)?
+        .tcp_nodelay(true)
+        .http2_adaptive_window(true)
         .buffer_size(buffer_config.buffer_size)
         .initial_connection_window_size(buffer_config.conn_window)
         .initial_stream_window_size(buffer_config.stream_window);

From 0604573f4ed3abb669e420b157dca292444d0333 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Fri, 14 Jun 2024 10:22:56 +0200
Subject: [PATCH 61/67] log if env config missing

---
 src/grpc_subscription_autoreconnect_tasks.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index 1ea0150..991a1b9 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -378,6 +378,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
 
 fn buffer_config_from_env() -> GeyserGrpcClientBufferConfig {
     if env::var("BUFFER_SIZE").is_err() || env::var("CONN_WINDOW").is_err() || env::var("STREAM_WINDOW").is_err() {
+        warn!("BUFFER_SIZE, CONN_WINDOW, STREAM_WINDOW not set; using default buffer config");
         return GeyserGrpcClientBufferConfig::default();
     }
 

From 172404720d3be325bbe023f58d6aab9f97363504 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 12 Aug 2024 11:46:32 +0200
Subject: [PATCH 62/67] compare account subscription with confirmation

---
 examples/stream_token_accounts.rs | 60 +++++++++++++++++++++++++++----
 src/lib.rs                        |  2 +-
 2 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index efdea13..4338d36 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -14,13 +14,14 @@ use solana_sdk::pubkey::Pubkey;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
 use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
-use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
+use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, map_commitment_level, Message};
 use tokio::time::{sleep, Duration};
 use tracing::field::debug;
 use tracing::warn;
 use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
 use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterSlots, SubscribeUpdate};
 use yellowstone_grpc_proto::prost::Message as _;
+use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
 
 
 const ENABLE_TIMESTAMP_TAGGING: bool = false;
@@ -51,9 +52,21 @@ pub async fn main() {
 
     info!("Write Block stream..");
 
-    let green_stream = create_geyser_reconnecting_stream(
+    let (exit_signal, exit_notify) = tokio::sync::broadcast::channel(1);
+    let (autoconnect_tx, mut accounts_rx) = tokio::sync::mpsc::channel(1000);
+
+    let green_stream = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
         token_accounts(),
+        autoconnect_tx.clone(),
+        exit_signal.subscribe(),
+    );
+
+    let green_blue = create_geyser_autoconnection_task_with_mpsc(
+        config.clone(),
+        token_accounts_finalized(),
+        autoconnect_tx.clone(),
+        exit_signal.subscribe(),
     );
 
     // owner x mint -> amount
@@ -70,8 +83,7 @@ pub async fn main() {
         let mut current_slot = 0;
 
 
-        let mut green_stream = pin!(green_stream);
-        while let Some(message) = green_stream.next().await {
+        while let Some(message) = accounts_rx.recv().await {
             match message {
                 Message::GeyserSubscribeUpdate(subscriber_update) => {
                     match subscriber_update.update_oneof {
@@ -139,7 +151,7 @@ pub async fn main() {
 
                                     let delta = (slot as i64) - (current_slot as i64);
                                     if delta > 1 {
-                                        info!("delta: {}", (slot as i64) - (current_slot as i64));
+                                        debug!("delta: {}", (slot as i64) - (current_slot as i64));
                                     }
 
                                     if slot != changing_slot && changing_slot != 0 {
@@ -198,6 +210,8 @@ pub async fn main() {
     sleep(Duration::from_secs(1800)).await;
 }
 
+
+
 pub fn token_accounts() -> SubscribeRequest {
     let mut accounts_subs = HashMap::new();
     accounts_subs.insert(
@@ -206,7 +220,7 @@ pub fn token_accounts() -> SubscribeRequest {
             account: vec![],
             // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
             owner:
-                spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
+            spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
         },
     );
@@ -231,3 +245,37 @@ pub fn token_accounts() -> SubscribeRequest {
     }
 }
 
+// find out if fialiized makes a difference wrt accounts
+pub fn token_accounts_finalized() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            owner:
+                spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
+            filters: vec![],
+        },
+    );
+
+
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert("client".to_string(), SubscribeRequestFilterSlots {
+        filter_by_commitment: Some(true),
+    });
+
+
+    SubscribeRequest {
+        slots: slots_subs,
+        accounts: accounts_subs,
+        transactions: HashMap::new(),
+        entry: Default::default(),
+        blocks: Default::default(),
+        blocks_meta: HashMap::new(),
+        commitment: Some(map_commitment_level(CommitmentConfig::finalized()).into()),
+        accounts_data_slice: Default::default(),
+        ping: None,
+    }
+}
+
diff --git a/src/lib.rs b/src/lib.rs
index 61a2e09..12e9d7b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -157,7 +157,7 @@ impl GeyserFilter {
     }
 }
 
-fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel {
+pub fn map_commitment_level(commitment_config: CommitmentConfig) -> CommitmentLevel {
     // solana_sdk -> yellowstone
     match commitment_config.commitment {
         solana_sdk::commitment_config::CommitmentLevel::Processed => {

From c239e1bb605add85bcf82dd0859d5088923e0069 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Mon, 12 Aug 2024 13:14:02 +0200
Subject: [PATCH 63/67] clarify

---
 examples/stream_token_accounts.rs | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index 4338d36..3b7ef2c 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -1,3 +1,4 @@
+use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use futures::{Stream, StreamExt};
 use log::{debug, info, trace};
@@ -7,6 +8,7 @@ use std::env;
 use std::pin::pin;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::time::Instant;
 use dashmap::DashMap;
 use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType, UiTokenAccount};
 use solana_account_decoder::parse_token::UiAccountState::Initialized;
@@ -55,14 +57,14 @@ pub async fn main() {
     let (exit_signal, exit_notify) = tokio::sync::broadcast::channel(1);
     let (autoconnect_tx, mut accounts_rx) = tokio::sync::mpsc::channel(1000);
 
-    let green_stream = create_geyser_autoconnection_task_with_mpsc(
+    let _jh_green = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
         token_accounts(),
         autoconnect_tx.clone(),
         exit_signal.subscribe(),
     );
 
-    let green_blue = create_geyser_autoconnection_task_with_mpsc(
+    let _jh_blue = create_geyser_autoconnection_task_with_mpsc(
         config.clone(),
         token_accounts_finalized(),
         autoconnect_tx.clone(),
@@ -82,6 +84,8 @@ pub async fn main() {
         let mut changing_slot = 0;
         let mut current_slot = 0;
 
+        let mut account_write_first_timestamp: HashMap<u64, Instant> = HashMap::new();
+
 
         while let Some(message) = accounts_rx.recv().await {
             match message {
@@ -95,6 +99,18 @@ pub async fn main() {
                             let account = update.account.unwrap();
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
                             let size = account.data.len() as u64;
+
+                            info!("got account write: {}", account.write_version);
+                            match account_write_first_timestamp.entry(account.write_version) {
+                                Entry::Occupied(o) => {
+                                    let first_timestamp = o.get();
+                                    info!("got second account update for same write version with delta of {:?}", first_timestamp.elapsed());
+                                }
+                                Entry::Vacant(v) => {
+                                    v.insert(Instant::now());
+                                }
+                            }
+
                             trace!("got account update: {} - {:?} - {} bytes",
                                 update.slot, account_pk, account.data.len());
 
@@ -103,7 +119,6 @@ pub async fn main() {
                                 info!("got account update: write_version={};timestamp_us={};slot={}", account.write_version, since_the_epoch.as_micros(), update.slot);
                             }
 
-
                             match parse_token(&account.data, Some(6)) {
                                 Ok(TokenAccountType::Account(account_ui)) => {
                                     // UiTokenAccount {
@@ -239,7 +254,7 @@ pub fn token_accounts() -> SubscribeRequest {
         entry: Default::default(),
         blocks: Default::default(),
         blocks_meta: HashMap::new(),
-        commitment: None,
+        commitment: Some(map_commitment_level(CommitmentConfig::processed()).into()),
         accounts_data_slice: Default::default(),
         ping: None,
     }
@@ -273,7 +288,7 @@ pub fn token_accounts_finalized() -> SubscribeRequest {
         entry: Default::default(),
         blocks: Default::default(),
         blocks_meta: HashMap::new(),
-        commitment: Some(map_commitment_level(CommitmentConfig::finalized()).into()),
+        commitment: Some(map_commitment_level(CommitmentConfig::confirmed()).into()),
         accounts_data_slice: Default::default(),
         ping: None,
     }

From 18729e8f2a7ec8d92a52097d8b179a1e17db50c5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 5 Nov 2024 17:01:05 +0100
Subject: [PATCH 64/67] dump

---
 examples/dump_slots_stream_samples.rs | 307 ++++++++++++++++++++++++++
 1 file changed, 307 insertions(+)
 create mode 100644 examples/dump_slots_stream_samples.rs

diff --git a/examples/dump_slots_stream_samples.rs b/examples/dump_slots_stream_samples.rs
new file mode 100644
index 0000000..e2973dd
--- /dev/null
+++ b/examples/dump_slots_stream_samples.rs
@@ -0,0 +1,307 @@
+///
+/// subsribe to grpc in multiple ways:
+/// - all slots and processed accounts in one subscription
+/// - only processed accounts
+/// - only confirmed accounts
+/// - only finalized accounts
+///
+/// we want to see if there is a difference in timing of "processed accounts" in the mix with slot vs "only processed accounts"
+use log::{info, warn};
+use solana_sdk::clock::Slot;
+use solana_sdk::commitment_config::{CommitmentConfig, CommitmentLevel};
+use std::collections::HashMap;
+use std::env;
+use std::str::FromStr;
+use std::time::SystemTime;
+
+use base64::Engine;
+use csv::ReaderBuilder;
+use itertools::Itertools;
+use solana_sdk::borsh0_10::try_from_slice_unchecked;
+/// This file mocks the core model of the RPC server.
+use solana_sdk::compute_budget;
+use solana_sdk::compute_budget::ComputeBudgetInstruction;
+use solana_sdk::hash::Hash;
+use solana_sdk::instruction::CompiledInstruction;
+use solana_sdk::message::v0::MessageAddressTableLookup;
+use solana_sdk::message::{v0, MessageHeader, VersionedMessage};
+use solana_sdk::pubkey::Pubkey;
+
+use solana_sdk::signature::Signature;
+use solana_sdk::transaction::TransactionError;
+use tokio::sync::mpsc::Receiver;
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, SubscribeUpdateSlot};
+
+use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
+use geyser_grpc_connector::{
+    map_commitment_level, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message,
+};
+use tokio::time::{sleep, Duration};
+use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
+use yellowstone_grpc_proto::geyser::SubscribeUpdate;
+use yellowstone_grpc_proto::prelude::SubscribeRequestFilterAccounts;
+
+fn start_all_slots_and_processed_accounts_consumer(mut slots_channel: Receiver<Message>) {
+    tokio::spawn(async move {
+        loop {
+            match slots_channel.recv().await {
+                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
+                    Some(UpdateOneof::Slot(update_slot)) => {
+                        let since_epoch_ms = SystemTime::now()
+                            .duration_since(SystemTime::UNIX_EPOCH)
+                            .unwrap()
+                            .as_millis();
+
+                        let short_status = match map_slot_status(&update_slot) {
+                            CommitmentLevel::Processed => "P",
+                            CommitmentLevel::Confirmed => "C",
+                            CommitmentLevel::Finalized => "F",
+                            _ => {
+                                panic!("unexpected commitment level")
+                            }
+                        };
+                        // DUMPSLOT 283356662,283356661,F,1723556492340
+                        info!(
+                            "MIXSLOT {},{:09},{},{}",
+                            update_slot.slot,
+                            update_slot.parent.unwrap_or(0),
+                            short_status,
+                            since_epoch_ms
+                        );
+                    }
+                    Some(UpdateOneof::Account(update_account)) => {
+                        let since_epoch_ms = SystemTime::now()
+                            .duration_since(SystemTime::UNIX_EPOCH)
+                            .unwrap()
+                            .as_millis();
+
+                        let account_info = update_account.account.unwrap();
+                        let slot = update_account.slot;
+                        let account_pk =
+                            Pubkey::new_from_array(account_info.pubkey.try_into().unwrap());
+                        let write_version = account_info.write_version;
+                        let data_len = account_info.data.len();
+                        // DUMPACCOUNT 283417593,HTQeo4GNbZfGY5G4fAkDr1S5xnz5qWXFgueRwgw53aU1,1332997857270,752,1723582355872
+                        info!(
+                            "MIXACCOUNT {},{},{},{},{}",
+                            slot, account_pk, write_version, data_len, since_epoch_ms
+                        );
+                    }
+                    None => {}
+                    _ => {}
+                },
+                None => {
+                    warn!("multiplexer channel closed - aborting");
+                    return;
+                }
+                Some(Message::Connecting(_)) => {}
+            }
+        }
+    });
+}
+
+// need to provide the commitment level used to filter the accounts
+fn start_account_same_level(
+    mut slots_channel: Receiver<Message>,
+    commitment_level: CommitmentLevel,
+) {
+    tokio::spawn(async move {
+        loop {
+            match slots_channel.recv().await {
+                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
+                    Some(UpdateOneof::Account(update_account)) => {
+                        let since_epoch_ms = SystemTime::now()
+                            .duration_since(SystemTime::UNIX_EPOCH)
+                            .unwrap()
+                            .as_millis();
+
+                        let account_info = update_account.account.unwrap();
+                        let slot = update_account.slot;
+                        let account_pk =
+                            Pubkey::new_from_array(account_info.pubkey.try_into().unwrap());
+                        let write_version = account_info.write_version;
+                        let data_len = account_info.data.len();
+
+                        let short_status = match commitment_level {
+                            CommitmentLevel::Processed => "P",
+                            CommitmentLevel::Confirmed => "C",
+                            CommitmentLevel::Finalized => "F",
+                            _ => {
+                                panic!("unexpected commitment level")
+                            }
+                        };
+
+                        // DUMPACCOUNT 283417593,HTQeo4GNbZfGY5G4fAkDr1S5xnz5qWXFgueRwgw53aU1,1332997857270,752,1723582355872
+                        info!(
+                            "DUMPACCOUNT {},{},{},{},{},{}",
+                            slot, short_status, account_pk, write_version, data_len, since_epoch_ms
+                        );
+                    }
+                    None => {}
+                    _ => {}
+                },
+                None => {
+                    warn!("multiplexer channel closed - aborting");
+                    return;
+                }
+                Some(Message::Connecting(_)) => {}
+            }
+        }
+    });
+}
+
+fn map_slot_status(
+    slot_update: &SubscribeUpdateSlot,
+) -> solana_sdk::commitment_config::CommitmentLevel {
+    use solana_sdk::commitment_config::CommitmentLevel as solanaCL;
+    use yellowstone_grpc_proto::geyser::CommitmentLevel as yCL;
+    yellowstone_grpc_proto::geyser::CommitmentLevel::try_from(slot_update.status)
+        .map(|v| match v {
+            yCL::Processed => solanaCL::Processed,
+            yCL::Confirmed => solanaCL::Confirmed,
+            yCL::Finalized => solanaCL::Finalized,
+        })
+        .expect("valid commitment level")
+}
+
+#[tokio::main]
+pub async fn main() {
+    tracing_subscriber::fmt::init();
+
+    let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green");
+    let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok();
+
+    info!(
+        "Using gRPC source {} ({})",
+        grpc_addr_green,
+        grpc_x_token_green.is_some()
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(5),
+        request_timeout: Duration::from_secs(5),
+        subscribe_timeout: Duration::from_secs(5),
+        receive_timeout: Duration::from_secs(5),
+    };
+
+    let green_config =
+        GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
+
+    // mix of (all) slots and processed accounts
+    let (autoconnect_tx, slots_accounts_rx) = tokio::sync::mpsc::channel(10);
+    let _green_stream_ah = create_geyser_autoconnection_task_with_mpsc(
+        green_config.clone(),
+        all_slots_and_processed_accounts_together(),
+        autoconnect_tx.clone(),
+    );
+
+    let (only_processed_accounts_tx, only_processed_accounts_rx) = tokio::sync::mpsc::channel(10);
+    let _accounts_processed_stream_ah = create_geyser_autoconnection_task_with_mpsc(
+        green_config.clone(),
+        accounts_at_level(CommitmentLevel::Processed),
+        only_processed_accounts_tx.clone(),
+    );
+
+    let (only_confirmed_accounts_tx, only_confirmed_accounts_rx) = tokio::sync::mpsc::channel(10);
+    let _accounts_confirmed_stream_ah = create_geyser_autoconnection_task_with_mpsc(
+        green_config.clone(),
+        accounts_at_level(CommitmentLevel::Confirmed),
+        only_confirmed_accounts_tx.clone(),
+    );
+
+    let (only_finalized_accounts_tx, only_finalized_accounts_rx) = tokio::sync::mpsc::channel(10);
+    let _accounts_finalized_stream_ah = create_geyser_autoconnection_task_with_mpsc(
+        green_config.clone(),
+        accounts_at_level(CommitmentLevel::Finalized),
+        only_finalized_accounts_tx.clone(),
+    );
+
+    start_all_slots_and_processed_accounts_consumer(slots_accounts_rx);
+    start_account_same_level(only_processed_accounts_rx, CommitmentLevel::Processed);
+    start_account_same_level(only_confirmed_accounts_rx, CommitmentLevel::Confirmed);
+    start_account_same_level(only_finalized_accounts_rx, CommitmentLevel::Finalized);
+
+    // "infinite" sleep
+    sleep(Duration::from_secs(3600 * 5)).await;
+}
+
+const RAYDIUM_AMM_PUBKEY: &'static str = "675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8";
+
+fn all_slots_and_processed_accounts_together() -> SubscribeRequest {
+    let mut slot_subs = HashMap::new();
+    slot_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterSlots {
+            // implies all slots
+            filter_by_commitment: None,
+        },
+    );
+    let mut account_subs = HashMap::new();
+    account_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![RAYDIUM_AMM_PUBKEY.to_string()],
+            filters: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        slots: slot_subs,
+        accounts: account_subs,
+        ping: None,
+        // implies "processed"
+        commitment: None,
+        ..Default::default()
+    }
+}
+
+fn accounts_at_level(commitment_level: CommitmentLevel) -> SubscribeRequest {
+    let mut account_subs = HashMap::new();
+    account_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![RAYDIUM_AMM_PUBKEY.to_string()],
+            filters: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        accounts: account_subs,
+        ping: None,
+        commitment: Some(map_commitment_level(CommitmentConfig {
+            commitment: commitment_level,
+        }) as i32),
+        ..Default::default()
+    }
+}
+
+#[test]
+fn parse_output() {
+    let data = "283360248,000000000,C,1723558000558";
+    let mut rdr = ReaderBuilder::new()
+        .has_headers(false)
+        .from_reader(data.as_bytes());
+
+    let all_records = rdr.records().collect_vec();
+    assert_eq!(1, all_records.len());
+    let record = all_records[0].as_ref().unwrap();
+
+    let slot: u64 = record[0].parse().unwrap();
+    let parent: Option<u64> = record[1]
+        .parse()
+        .ok()
+        .and_then(|v| if v == 0 { None } else { Some(v) });
+    let status = match record[2].to_string().as_str() {
+        "P" => CommitmentLevel::Processed,
+        "C" => CommitmentLevel::Confirmed,
+        "F" => CommitmentLevel::Finalized,
+        _ => panic!("invalid commitment level"),
+    };
+
+    assert_eq!(283360248, slot);
+    assert_eq!(None, parent);
+    assert_eq!(CommitmentLevel::Confirmed, status);
+}

From 8574783f513fa3db11948eac57c92ae8be9b9379 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 5 Nov 2024 17:01:47 +0100
Subject: [PATCH 65/67] wip

---
 examples/dump_txs_stream_samples.rs          | 116 +++++++++++++++++++
 examples/stream_token_accounts.rs            |  23 ++--
 examples/stream_vote_transactions.rs         |   1 -
 src/grpc_subscription_autoreconnect_tasks.rs |   1 +
 4 files changed, 133 insertions(+), 8 deletions(-)
 create mode 100644 examples/dump_txs_stream_samples.rs

diff --git a/examples/dump_txs_stream_samples.rs b/examples/dump_txs_stream_samples.rs
new file mode 100644
index 0000000..0dbab71
--- /dev/null
+++ b/examples/dump_txs_stream_samples.rs
@@ -0,0 +1,116 @@
+use log::{info, warn};
+use solana_sdk::clock::Slot;
+use solana_sdk::commitment_config::{CommitmentConfig, CommitmentLevel};
+use std::collections::HashMap;
+use std::env;
+use std::str::FromStr;
+use std::time::SystemTime;
+
+use base64::Engine;
+use csv::ReaderBuilder;
+use itertools::Itertools;
+use solana_sdk::borsh0_10::try_from_slice_unchecked;
+/// This file mocks the core model of the RPC server.
+use solana_sdk::compute_budget;
+use solana_sdk::compute_budget::ComputeBudgetInstruction;
+use solana_sdk::hash::Hash;
+use solana_sdk::instruction::CompiledInstruction;
+use solana_sdk::message::v0::MessageAddressTableLookup;
+use solana_sdk::message::{v0, MessageHeader, VersionedMessage};
+use solana_sdk::pubkey::Pubkey;
+
+use solana_sdk::signature::Signature;
+use solana_sdk::transaction::TransactionError;
+use tokio::sync::broadcast;
+use tokio::sync::mpsc::Receiver;
+use yellowstone_grpc_proto::geyser::{SubscribeRequest, SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, SubscribeUpdateSlot};
+
+use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
+use geyser_grpc_connector::{
+    map_commitment_level, GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message,
+};
+use tokio::time::{sleep, Duration};
+use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
+use yellowstone_grpc_proto::geyser::SubscribeUpdate;
+use yellowstone_grpc_proto::prelude::SubscribeRequestFilterAccounts;
+
+
+
+#[tokio::main]
+pub async fn main() {
+    tracing_subscriber::fmt::init();
+
+    let grpc_addr_green = env::var("GRPC_ADDR").expect("need grpc url for green");
+    let grpc_x_token_green = env::var("GRPC_X_TOKEN").ok();
+
+    let (_foo, exit_notify) = broadcast::channel(1);
+
+    info!(
+        "Using gRPC source {} ({})",
+        grpc_addr_green,
+        grpc_x_token_green.is_some()
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(5),
+        request_timeout: Duration::from_secs(5),
+        subscribe_timeout: Duration::from_secs(5),
+        receive_timeout: Duration::from_secs(5),
+    };
+
+    let green_config =
+        GrpcSourceConfig::new(grpc_addr_green, grpc_x_token_green, None, timeouts.clone());
+
+
+    let (autoconnect_tx, mut transactions_rx) = tokio::sync::mpsc::channel(10);
+    let _tx_source_ah = create_geyser_autoconnection_task_with_mpsc(
+        green_config.clone(),
+        jupyter_trades(),
+        autoconnect_tx.clone(),
+        exit_notify,
+    );
+
+    loop {
+        let message = transactions_rx.recv().await;
+        match message {
+            Some(Message::GeyserSubscribeUpdate(update)) =>
+                match update.update_oneof {
+                    Some(UpdateOneof::Transaction(update)) => {
+                        let tx = update.transaction.unwrap();
+                        let sig = Signature::try_from(tx.signature.as_slice()).unwrap();
+                        info!("tx {}", sig);
+                    }
+                    _ => {} // FIXME
+                },
+            _ => {} // FIXME
+        }
+    }
+
+
+}
+
+
+fn jupyter_trades() -> SubscribeRequest {
+    let mut transaction_subs = HashMap::new();
+    transaction_subs.insert("client".to_string(),
+                            SubscribeRequestFilterTransactions {
+                                vote: Some(false),
+                                failed: Some(false),
+                                signature: None,
+                                account_include: vec![
+                                    "JUP6LkbZbjS1jKKwapdHNy74zcZ3tLUZoi5QNyVTaV4".to_string(),
+                                ],
+                                account_exclude: vec![],
+                                account_required: vec![],
+                            });
+
+    SubscribeRequest {
+        transactions: transaction_subs,
+        ping: None,
+        commitment: Some(map_commitment_level(CommitmentConfig {
+            commitment: CommitmentLevel::Confirmed,
+        }) as i32),
+        ..Default::default()
+    }
+}
\ No newline at end of file
diff --git a/examples/stream_token_accounts.rs b/examples/stream_token_accounts.rs
index 3b7ef2c..db63338 100644
--- a/examples/stream_token_accounts.rs
+++ b/examples/stream_token_accounts.rs
@@ -2,7 +2,7 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use futures::{Stream, StreamExt};
 use log::{debug, info, trace};
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Clock, Slot};
 use solana_sdk::commitment_config::CommitmentConfig;
 use std::env;
 use std::pin::pin;
@@ -13,6 +13,7 @@ use dashmap::DashMap;
 use solana_account_decoder::parse_token::{parse_token, spl_token_ids, TokenAccountType, UiTokenAccount};
 use solana_account_decoder::parse_token::UiAccountState::Initialized;
 use solana_sdk::pubkey::Pubkey;
+use solana_sdk::sysvar::clock;
 
 use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
 use geyser_grpc_connector::grpcmultiplex_fastestwins::FromYellowstoneExtractor;
@@ -100,6 +101,14 @@ pub async fn main() {
                             let account_pk = Pubkey::try_from(account.pubkey).unwrap();
                             let size = account.data.len() as u64;
 
+                            info!("got account update: {} - {:?} - {} bytes",
+                                update.slot, account_pk, account.data.len());
+
+                            if clock::id() == account_pk {
+                                let clock: Clock = bincode::deserialize(&account.data).unwrap();
+                                info!("clock: {:#?}", clock);
+                            }
+
                             info!("got account write: {}", account.write_version);
                             match account_write_first_timestamp.entry(account.write_version) {
                                 Entry::Occupied(o) => {
@@ -232,10 +241,10 @@ pub fn token_accounts() -> SubscribeRequest {
     accounts_subs.insert(
         "client".to_string(),
         SubscribeRequestFilterAccounts {
-            account: vec![],
-            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
-            owner:
-            spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
+            // account: vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            account: vec![clock::id().to_string()],
+            owner: vec![],
+            // spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
         },
     );
@@ -266,8 +275,8 @@ pub fn token_accounts_finalized() -> SubscribeRequest {
     accounts_subs.insert(
         "client".to_string(),
         SubscribeRequestFilterAccounts {
-            account: vec![],
-            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            account:
+            vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
             owner:
                 spl_token_ids().iter().map(|pubkey| pubkey.to_string()).collect(),
             filters: vec![],
diff --git a/examples/stream_vote_transactions.rs b/examples/stream_vote_transactions.rs
index 462ac4a..8fd37c3 100644
--- a/examples/stream_vote_transactions.rs
+++ b/examples/stream_vote_transactions.rs
@@ -144,4 +144,3 @@ pub fn transaction_filter() -> SubscribeRequest {
         ping: None,
     }
 }
-
diff --git a/src/grpc_subscription_autoreconnect_tasks.rs b/src/grpc_subscription_autoreconnect_tasks.rs
index 991a1b9..494be04 100644
--- a/src/grpc_subscription_autoreconnect_tasks.rs
+++ b/src/grpc_subscription_autoreconnect_tasks.rs
@@ -231,6 +231,7 @@ pub fn create_geyser_autoconnection_task_with_mpsc(
                 ConnectionState::FatalError(_attempt, reason) => match reason {
                     FatalErrorReason::DownstreamChannelClosed => {
                         warn!("downstream closed - aborting");
+                        // TODO break 'main_loop instead of returning
                         return;
                     }
                     FatalErrorReason::ConfigurationError => {

From c71cbbe05ad925971d612f5cb28bc2826d87822d Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Tue, 12 Nov 2024 17:47:20 +0100
Subject: [PATCH 66/67] accounts stream from da11 ledger-tool

---
 examples/subscribe_accounts.rs | 193 +++++++++++++++++++++++++++++++++
 1 file changed, 193 insertions(+)
 create mode 100644 examples/subscribe_accounts.rs

diff --git a/examples/subscribe_accounts.rs b/examples/subscribe_accounts.rs
new file mode 100644
index 0000000..09c170f
--- /dev/null
+++ b/examples/subscribe_accounts.rs
@@ -0,0 +1,193 @@
+//
+// ```
+// ssh eclipse-rpc -Nv
+// ```
+//
+
+use futures::{Stream, StreamExt};
+use itertools::Itertools;
+use log::{debug, info};
+use solana_account_decoder::parse_token::spl_token_ids;
+use solana_sdk::clock::{Slot, UnixTimestamp};
+use solana_sdk::commitment_config::CommitmentConfig;
+use solana_sdk::hash::{hash, Hash};
+use solana_sdk::pubkey::Pubkey;
+use std::cmp::min;
+use std::collections::{HashMap, VecDeque};
+use std::pin::pin;
+use std::str::FromStr;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{Instant, SystemTime, UNIX_EPOCH};
+use std::{env, iter};
+use tokio::sync::mpsc::Receiver;
+
+use geyser_grpc_connector::grpc_subscription_autoreconnect_streams::create_geyser_reconnecting_stream;
+use geyser_grpc_connector::grpc_subscription_autoreconnect_tasks::create_geyser_autoconnection_task_with_mpsc;
+use geyser_grpc_connector::grpcmultiplex_fastestwins::{
+    create_multiplexed_stream, FromYellowstoneExtractor,
+};
+use geyser_grpc_connector::{GeyserFilter, GrpcConnectionTimeouts, GrpcSourceConfig, Message};
+use tokio::time::{sleep, Duration};
+use tracing::{trace, warn};
+use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof;
+use yellowstone_grpc_proto::geyser::{
+    SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterBlocksMeta,
+    SubscribeRequestFilterSlots, SubscribeUpdate,
+};
+use yellowstone_grpc_proto::prost::Message as _;
+
+type AtomicSlot = Arc<AtomicU64>;
+
+#[tokio::main]
+pub async fn main() {
+    // RUST_LOG=info,stream_blocks_mainnet=debug,geyser_grpc_connector=trace
+    tracing_subscriber::fmt::init();
+    // console_subscriber::init();
+
+    let grpc_addr = env::var("GRPC_ADDR").expect("need grpc url");
+    let grpc_x_token = env::var("GRPC_X_TOKEN").ok();
+
+    info!(
+        "Using grpc source on {} ({})",
+        grpc_addr,
+        grpc_x_token.is_some()
+    );
+
+    let timeouts = GrpcConnectionTimeouts {
+        connect_timeout: Duration::from_secs(25),
+        request_timeout: Duration::from_secs(25),
+        subscribe_timeout: Duration::from_secs(25),
+        receive_timeout: Duration::from_secs(25),
+    };
+
+    let config = GrpcSourceConfig::new(grpc_addr, grpc_x_token, None, timeouts.clone());
+
+    let (autoconnect_tx, geyser_messages_rx) = tokio::sync::mpsc::channel(10);
+    let (_exit_tx, exit_rx) = tokio::sync::broadcast::channel::<()>(1);
+
+    let _all_accounts = create_geyser_autoconnection_task_with_mpsc(
+        config.clone(),
+        all_accounts(),
+        autoconnect_tx.clone(),
+        exit_rx.resubscribe(),
+    );
+
+    let current_processed_slot = AtomicSlot::default();
+    start_tracking_account_consumer(geyser_messages_rx, current_processed_slot.clone());
+
+    // "infinite" sleep
+    sleep(Duration::from_secs(1800)).await;
+}
+
+// note: this keeps track of lot of data and might blow up memory
+fn start_tracking_account_consumer(
+    mut geyser_messages_rx: Receiver<Message>,
+    current_processed_slot: Arc<AtomicU64>,
+) {
+    tokio::spawn(async move {
+        loop {
+            match geyser_messages_rx.recv().await {
+                Some(Message::GeyserSubscribeUpdate(update)) => match update.update_oneof {
+                    Some(UpdateOneof::Account(update)) => {
+                        let started_at = Instant::now();
+                        let now = SystemTime::now();
+                        let account_info = update.account.unwrap();
+                        let account_pk = Pubkey::try_from(account_info.pubkey).unwrap();
+                        let account_owner_pk = Pubkey::try_from(account_info.owner).unwrap();
+                        // note: slot is referencing the block that is just built while the slot number reported from BlockMeta/Slot uses the slot after the block is built
+                        let slot = update.slot;
+                        let account_receive_time = get_epoch_sec();
+
+                        info!(
+                            "Account update: slot: {}, account_pk: {}, account_owner_pk: {}, account_receive_time: {}",
+                            slot, account_pk, account_owner_pk, account_receive_time
+                        );
+                    }
+                    None => {}
+                    _ => {}
+                },
+                None => {
+                    log::warn!("multiplexer channel closed - aborting");
+                    return;
+                }
+                Some(Message::Connecting(_)) => {}
+            }
+        }
+    });
+}
+
+fn get_epoch_sec() -> UnixTimestamp {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs() as UnixTimestamp
+}
+
+pub fn token_accounts() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            // vec!["4DoNfFBfF7UokCC2FQzriy7yHK6DY6NVdYpuekQ5pRgg".to_string()],
+            owner: spl_token_ids()
+                .iter()
+                .map(|pubkey| pubkey.to_string())
+                .collect(),
+            filters: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        accounts: accounts_subs,
+        ..Default::default()
+    }
+}
+
+pub fn all_accounts_and_blocksmeta() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![],
+            filters: vec![],
+        },
+    );
+
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterSlots {
+            filter_by_commitment: Some(true),
+        },
+    );
+
+    let mut blocks_meta_subs = HashMap::new();
+    blocks_meta_subs.insert("client".to_string(), SubscribeRequestFilterBlocksMeta {});
+
+    SubscribeRequest {
+        slots: slots_subs,
+        accounts: accounts_subs,
+        blocks_meta: blocks_meta_subs,
+        ..Default::default()
+    }
+}
+
+pub fn all_accounts() -> SubscribeRequest {
+    let mut accounts_subs = HashMap::new();
+    accounts_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterAccounts {
+            account: vec![],
+            owner: vec![],
+            filters: vec![],
+        },
+    );
+
+    SubscribeRequest {
+        accounts: accounts_subs,
+        ..Default::default()
+    }
+}

From 6e9761692e37d7454d5f4d7a224bc56b930269a5 Mon Sep 17 00:00:00 2001
From: GroovieGermanikus <groovie@mango.markets>
Date: Thu, 14 Nov 2024 12:13:02 +0100
Subject: [PATCH 67/67] subscripbe accounts

---
 examples/subscribe_accounts.rs | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/examples/subscribe_accounts.rs b/examples/subscribe_accounts.rs
index 09c170f..8cd64d8 100644
--- a/examples/subscribe_accounts.rs
+++ b/examples/subscribe_accounts.rs
@@ -191,3 +191,20 @@ pub fn all_accounts() -> SubscribeRequest {
         ..Default::default()
     }
 }
+
+
+pub fn slots() -> SubscribeRequest {
+    let mut slots_subs = HashMap::new();
+    slots_subs.insert(
+        "client".to_string(),
+        SubscribeRequestFilterSlots {
+            filter_by_commitment: None,
+        },
+    );
+
+    SubscribeRequest {
+        slots: slots_subs,
+        ..Default::default()
+    }
+}
+