From 71d81ddea74deeeae6d1666a55f0939cfe56472c Mon Sep 17 00:00:00 2001 From: Lyn <0xlynett@proton.me> Date: Fri, 19 Jul 2024 16:29:45 +0100 Subject: [PATCH 1/8] use build args in dockerfile --- Dockerfile | 6 +++--- README.md | 30 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd6d6b756..2f1f5822d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM debian:12-slim AS downloader +ARG VERSION WORKDIR /tmp/download RUN apt-get update -RUN apt-get install wget curl openssl jq unzip -y +RUN apt-get install unzip -y -ADD https://api.github.com/repos/kinode-dao/kinode/releases releases.json -RUN wget "https://github.com/kinode-dao/kinode/releases/download/$(cat releases.json | jq -r '.[0].tag_name')/kinode-x86_64-unknown-linux-gnu.zip" +ADD "https://github.com/kinode-dao/kinode/releases/download/${VERSION}/kinode-x86_64-unknown-linux-gnu.zip" kinode-x86_64-unknown-linux-gnu.zip RUN unzip kinode-x86_64-unknown-linux-gnu.zip FROM debian:12-slim diff --git a/README.md b/README.md index 2ffece5c4..3e30146b7 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@

- Kinode is a general-purpose sovereign cloud computer, built for crypto. This repo contains the core runtime and processes. @@ -17,10 +16,10 @@ Then follow the instructions to [install it](https://book.kinode.org/install.htm If you have questions, join the [Kinode discord](https://discord.gg/TCgdca5Bjt) and drop us a line in `#dev-support`. - ## Setup On certain operating systems, you may need to install these dependencies if they are not already present: + - openssl-sys: https://docs.rs/crate/openssl-sys/0.9.19 - libclang 5.0: https://rust-lang.github.io/rust-bindgen/requirements.html @@ -54,6 +53,7 @@ No security audits of this crate have ever been performed. This software is unde Make sure not to use the same home directory for two nodes at once! You can use any name for the home directory: here we just use `home`. The `--` here separates cargo arguments from binary arguments. TODO: document feature flags in `--simulation-mode` + ```bash # OPTIONAL: --release flag cargo +nightly run -p kinode -- home @@ -66,6 +66,7 @@ On boot you will be prompted to navigate to `localhost:8080` (or whatever HTTP p By default, a node will use the [hardcoded providers](./kinode/src/eth/default_providers_mainnet.json) for the network it is booted on. A node can use a WebSockets RPC URL directly, or use another Kinode as a relay point. To adjust the providers a node uses, just create and modify the `.eth_providers` file in the node's home folder (set at boot). See the Kinode Book for more docs, and see the [default providers file here](./kinode/src/eth/default_providers_mainnet.json) for a template to create `.eth_providers`. You may also add a RPC provider or otherwise modify your configuration by sending messages from the terminal to the `eth:distro:sys` process. You can get one for free at `alchemy.com`. Use this message format to add a provider -- this will make your node's performance better when accessing a blockchain: + ``` m our@eth:distro:sys '{"AddProvider": {"chain_id": , "trusted": true, "provider": {"RpcUrl": ""}}}' ``` @@ -121,18 +122,18 @@ The `sys` publisher is not a real node ID, but it's also not a special case valu - CTRL+R to search history, CTRL+R again to toggle through search results, CTRL+G to cancel search - `m
''`: send an inter-process message.
is formatted as @. is formatted as ::. JSON containing spaces must be wrapped in single-quotes (`''`). - - Example: `m our@eth:distro:sys "SetPublic" -a 5` - - the '-a' flag is used to expect a response with a given timeout - - `our` will always be interpolated by the system as your node's name + - Example: `m our@eth:distro:sys "SetPublic" -a 5` + - the '-a' flag is used to expect a response with a given timeout + - `our` will always be interpolated by the system as your node's name - `hi `: send a text message to another node's command line. - - Example: `hi ben.os hello world` + - Example: `hi ben.os hello world` - `top `: display kernel debugging info about a process. Leave the process ID blank to display info about all processes and get the total number of running processes. - - Example: `top net:distro:sys` - - Example: `top` + - Example: `top net:distro:sys` + - Example: `top` - `cat `: print the contents of a file in the terminal - - Example: `cat /terminal:sys/pkg/scripts.json` + - Example: `cat /terminal:sys/pkg/scripts.json` - `echo `: print `text` to the terminal - - Example: `echo foo` + - Example: `echo foo` - `net_diagnostics`: print some useful networking diagnostic data - `peers`: print the peers the node currently hold connections with - `peer `: print the peer's PKI info, if it exists @@ -146,12 +147,15 @@ The image includes EXPOSE directives for TCP port `8080` and TCP port `9000`. Po If you are running a direct node, you must map port `9000` to the same port on the host and on your router. Otherwise, your Kinode will not be able to connect to the rest of the network as connection info is written to the chain, and this information is based on the view from inside the Docker container. To build a local Docker image, run the following command in this project root. -``` -docker build -t 0xlynett/kinode . + +```bash +# The `VERSION` may be replaced with the tag of a GitHub release +docker build -t 0xlynett/kinode . --build-arg VERSION=v0.8.6 ``` For example: -``` + +```bash docker volume create kinode-volume docker run -d -p 8080:8080 -it --name my-kinode \ From 6298afdd5808b479c787bfc73b2c48c915773a47 Mon Sep 17 00:00:00 2001 From: Lyn <0xlynett@proton.me> Date: Fri, 19 Jul 2024 16:32:37 +0100 Subject: [PATCH 2/8] return formatting to normal --- README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3e30146b7..5d3350fbb 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@

+ Kinode is a general-purpose sovereign cloud computer, built for crypto. This repo contains the core runtime and processes. @@ -16,10 +17,10 @@ Then follow the instructions to [install it](https://book.kinode.org/install.htm If you have questions, join the [Kinode discord](https://discord.gg/TCgdca5Bjt) and drop us a line in `#dev-support`. + ## Setup On certain operating systems, you may need to install these dependencies if they are not already present: - - openssl-sys: https://docs.rs/crate/openssl-sys/0.9.19 - libclang 5.0: https://rust-lang.github.io/rust-bindgen/requirements.html @@ -53,7 +54,6 @@ No security audits of this crate have ever been performed. This software is unde Make sure not to use the same home directory for two nodes at once! You can use any name for the home directory: here we just use `home`. The `--` here separates cargo arguments from binary arguments. TODO: document feature flags in `--simulation-mode` - ```bash # OPTIONAL: --release flag cargo +nightly run -p kinode -- home @@ -66,7 +66,6 @@ On boot you will be prompted to navigate to `localhost:8080` (or whatever HTTP p By default, a node will use the [hardcoded providers](./kinode/src/eth/default_providers_mainnet.json) for the network it is booted on. A node can use a WebSockets RPC URL directly, or use another Kinode as a relay point. To adjust the providers a node uses, just create and modify the `.eth_providers` file in the node's home folder (set at boot). See the Kinode Book for more docs, and see the [default providers file here](./kinode/src/eth/default_providers_mainnet.json) for a template to create `.eth_providers`. You may also add a RPC provider or otherwise modify your configuration by sending messages from the terminal to the `eth:distro:sys` process. You can get one for free at `alchemy.com`. Use this message format to add a provider -- this will make your node's performance better when accessing a blockchain: - ``` m our@eth:distro:sys '{"AddProvider": {"chain_id": , "trusted": true, "provider": {"RpcUrl": ""}}}' ``` @@ -122,18 +121,18 @@ The `sys` publisher is not a real node ID, but it's also not a special case valu - CTRL+R to search history, CTRL+R again to toggle through search results, CTRL+G to cancel search - `m
''`: send an inter-process message.
is formatted as @. is formatted as ::. JSON containing spaces must be wrapped in single-quotes (`''`). - - Example: `m our@eth:distro:sys "SetPublic" -a 5` - - the '-a' flag is used to expect a response with a given timeout - - `our` will always be interpolated by the system as your node's name + - Example: `m our@eth:distro:sys "SetPublic" -a 5` + - the '-a' flag is used to expect a response with a given timeout + - `our` will always be interpolated by the system as your node's name - `hi `: send a text message to another node's command line. - - Example: `hi ben.os hello world` + - Example: `hi ben.os hello world` - `top `: display kernel debugging info about a process. Leave the process ID blank to display info about all processes and get the total number of running processes. - - Example: `top net:distro:sys` - - Example: `top` + - Example: `top net:distro:sys` + - Example: `top` - `cat `: print the contents of a file in the terminal - - Example: `cat /terminal:sys/pkg/scripts.json` + - Example: `cat /terminal:sys/pkg/scripts.json` - `echo `: print `text` to the terminal - - Example: `echo foo` + - Example: `echo foo` - `net_diagnostics`: print some useful networking diagnostic data - `peers`: print the peers the node currently hold connections with - `peer `: print the peer's PKI info, if it exists @@ -147,7 +146,6 @@ The image includes EXPOSE directives for TCP port `8080` and TCP port `9000`. Po If you are running a direct node, you must map port `9000` to the same port on the host and on your router. Otherwise, your Kinode will not be able to connect to the rest of the network as connection info is written to the chain, and this information is based on the view from inside the Docker container. To build a local Docker image, run the following command in this project root. - ```bash # The `VERSION` may be replaced with the tag of a GitHub release docker build -t 0xlynett/kinode . --build-arg VERSION=v0.8.6 @@ -161,4 +159,4 @@ docker volume create kinode-volume docker run -d -p 8080:8080 -it --name my-kinode \ --mount type=volume,source=kinode-volume,destination=/kinode-home \ 0xlynett/kinode -``` +``` \ No newline at end of file From 5106f43c56d51024b9c8ed9fa3c786ac0be9bd66 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 09:44:59 -0700 Subject: [PATCH 3/8] bump versions to 0.8.7 --- Cargo.toml | 2 +- kinode/Cargo.toml | 2 +- lib/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e37cf88ea..f4bb93eef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kinode_lib" authors = ["KinodeDAO"] -version = "0.8.6" +version = "0.8.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" diff --git a/kinode/Cargo.toml b/kinode/Cargo.toml index c0f5ee856..5238cca40 100644 --- a/kinode/Cargo.toml +++ b/kinode/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kinode" authors = ["KinodeDAO"] -version = "0.8.6" +version = "0.8.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index b79fc8a7b..909693c2c 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lib" authors = ["KinodeDAO"] -version = "0.8.6" +version = "0.8.7" edition = "2021" description = "A general-purpose sovereign cloud computing platform" homepage = "https://kinode.org" From 2259449321f0613c37db929886150d6dce0a4a3c Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 19 Jul 2024 14:46:10 -0700 Subject: [PATCH 4/8] eth: avoid ws provider race condition by checking before replacing --- Cargo.lock | 6 +++--- kinode/src/eth/mod.rs | 7 +++++-- kinode/src/eth/subscription.rs | 8 ++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a695cf9c5..da4ff72a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3203,7 +3203,7 @@ dependencies = [ [[package]] name = "kinode" -version = "0.8.6" +version = "0.8.7" dependencies = [ "aes-gcm", "alloy", @@ -3259,7 +3259,7 @@ dependencies = [ [[package]] name = "kinode_lib" -version = "0.8.6" +version = "0.8.7" dependencies = [ "lib", ] @@ -3376,7 +3376,7 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "lib" -version = "0.8.6" +version = "0.8.7" dependencies = [ "alloy", "kit", diff --git a/kinode/src/eth/mod.rs b/kinode/src/eth/mod.rs index 53d640aad..190e97395 100644 --- a/kinode/src/eth/mod.rs +++ b/kinode/src/eth/mod.rs @@ -626,8 +626,11 @@ async fn fulfill_request( is_replacement_successful = false; return (); }; - aps.urls.remove(index); - aps.urls.insert(0, url_provider.clone()); + let old_provider = aps.urls.remove(index); + match old_provider.pubsub { + None => aps.urls.insert(0, url_provider.clone()), + Some(_) => aps.urls.insert(0, old_provider), + } }); if !is_replacement_successful { verbose_print( diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 3c7e37973..ccf554d5e 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -232,8 +232,12 @@ async fn build_subscription( is_replacement_successful = false; return (); }; - aps.urls.remove(index); - aps.urls.insert(0, url_provider.clone()); + let old_provider = aps.urls.remove(index); + match old_provider.pubsub { + None => aps.urls.insert(0, url_provider.clone()), + Some(_) => aps.urls.insert(0, old_provider), + } + }); if !is_replacement_successful { verbose_print( From e876891e1cf576d81efba9d34d3f5cd72f1bbc89 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 21:46:37 +0000 Subject: [PATCH 5/8] Format Rust code using rustfmt --- kinode/src/eth/subscription.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index ccf554d5e..781f84415 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -237,7 +237,6 @@ async fn build_subscription( None => aps.urls.insert(0, url_provider.clone()), Some(_) => aps.urls.insert(0, old_provider), } - }); if !is_replacement_successful { verbose_print( From 81fa748d0a5f515ff41f0a7d780dc47af51ee0de Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 8 Aug 2024 16:05:13 +0300 Subject: [PATCH 6/8] eth provider: try and get more diagnostics to provider term --- kinode/src/eth/mod.rs | 55 ++++++++++++++++++++++++---------- kinode/src/eth/subscription.rs | 1 - 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/kinode/src/eth/mod.rs b/kinode/src/eth/mod.rs index 190e97395..b23742d2f 100644 --- a/kinode/src/eth/mod.rs +++ b/kinode/src/eth/mod.rs @@ -354,19 +354,25 @@ async fn handle_message( Ok(EthSub { id, .. }) => id, Err(EthSubError { id, .. }) => id, }; - if let Some(sub_map) = state.active_subscriptions.get(&rsvp) { - if let Some(ActiveSub::Remote { - provider_node, - sender, - .. - }) = sub_map.get(&sub_id) - { - if provider_node == &km.source.node { - if let Ok(()) = sender.send(eth_sub_result).await { - // successfully sent a subscription update from a - // remote provider to one of our processes - return Ok(()); + if let Some(mut sub_map) = state.active_subscriptions.get_mut(&rsvp) { + if let Some(sub) = sub_map.get(&sub_id) { + if let ActiveSub::Remote { + provider_node, + sender, + .. + } = sub + { + if provider_node == &km.source.node { + if let Ok(()) = sender.send(eth_sub_result).await { + // successfully sent a subscription update from a + // remote provider to one of our processes + return Ok(()); + } } + // failed to send subscription update to process, + // unsubscribe from provider and close + sub.close(sub_id, state).await; + sub_map.remove(&sub_id); } } } @@ -479,12 +485,29 @@ async fn handle_eth_action( .await; } EthAction::UnsubscribeLogs(sub_id) => { - let mut sub_map = state - .active_subscriptions - .entry(km.source.clone()) - .or_insert(HashMap::new()); + let Some(mut sub_map) = state.active_subscriptions.get_mut(&km.source) else { + verbose_print( + &state.print_tx, + "eth: got unsubscribe but no matching subscription found", + ) + .await; + error_message( + &state.our, + km.id, + km.source, + EthError::MalformedRequest, + &state.send_to_loop, + ) + .await; + return Ok(()); + }; if let Some(sub) = sub_map.remove(&sub_id) { sub.close(sub_id, state).await; + verbose_print( + &state.print_tx, + &format!("eth: closed subscription {} for {}", sub_id, km.source.node), + ) + .await; kernel_message( &state.our, km.id, diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 781f84415..92a3216a1 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -113,7 +113,6 @@ pub async fn create_new_subscription( let (keepalive_err_sender, keepalive_err_receiver) = tokio::sync::mpsc::channel(1); response_channels.insert(keepalive_km_id, keepalive_err_sender); - let response_channels = response_channels.clone(); subs.insert( remote_sub_id, ActiveSub::Remote { From 19a750b5e5c86f3fd6fe16d4e0213c76dcd61644 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 8 Aug 2024 18:15:24 +0300 Subject: [PATCH 7/8] eth: manually unsubscribe if we break remote_subscription due to error --- kinode/src/eth/subscription.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 92a3216a1..875de5038 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -470,7 +470,7 @@ async fn maintain_remote_subscription( true, Some(30), IncomingReq::SubKeepalive(remote_sub_id), - &send_to_loop, + send_to_loop, ).await; } _incoming = net_error_rx.recv() => { @@ -487,6 +487,23 @@ async fn maintain_remote_subscription( } } }; + // tell provider node we don't need their services anymore + // (in case they did not close the subscription on their side, + // such as in the 2-hour timeout case) + kernel_message( + our, + rand::random(), + Address { + node: provider_node.to_string(), + process: ETH_PROCESS_ID.clone(), + }, + None, + true, + None, + EthAction::UnsubscribeLogs(sub_id), + send_to_loop, + ) + .await; active_subscriptions .entry(target.clone()) .and_modify(|sub_map| { From a288fc1fd4f508e0d4ca16d40afc55c7d0281295 Mon Sep 17 00:00:00 2001 From: dr-frmr Date: Thu, 8 Aug 2024 19:12:13 +0300 Subject: [PATCH 8/8] more diagnostics, fixes --- kinode/src/eth/mod.rs | 37 +++++++++++++++++++++++++++------- kinode/src/eth/subscription.rs | 2 +- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/kinode/src/eth/mod.rs b/kinode/src/eth/mod.rs index b23742d2f..cec64a51a 100644 --- a/kinode/src/eth/mod.rs +++ b/kinode/src/eth/mod.rs @@ -322,6 +322,7 @@ async fn handle_message( ) .await; } + Ok(()) } Message::Request(req) => { let timeout = req.expects_response.unwrap_or(60); @@ -330,7 +331,7 @@ async fn handle_message( }; match req { IncomingReq::EthAction(eth_action) => { - return handle_eth_action(state, km, timeout, eth_action).await; + handle_eth_action(state, km, timeout, eth_action).await } IncomingReq::EthConfigAction(eth_config_action) => { kernel_message( @@ -344,10 +345,16 @@ async fn handle_message( &state.send_to_loop, ) .await; + Ok(()) } IncomingReq::EthSubResult(eth_sub_result) => { // forward this to rsvp, if we have the sub id in our active subs let Some(rsvp) = km.rsvp else { + verbose_print( + &state.print_tx, + "eth: got eth_sub_result with no rsvp, ignoring", + ) + .await; return Ok(()); // no rsvp, no need to forward }; let sub_id = match eth_sub_result { @@ -371,8 +378,14 @@ async fn handle_message( } // failed to send subscription update to process, // unsubscribe from provider and close + verbose_print( + &state.print_tx, + "eth: got eth_sub_result but provider node did not match or local sub was already closed", + ) + .await; sub.close(sub_id, state).await; sub_map.remove(&sub_id); + return Ok(()); } } } @@ -380,13 +393,16 @@ async fn handle_message( // so they can stop sending us updates verbose_print( &state.print_tx, - "eth: got eth_sub_result but no matching sub found, unsubscribing", + &format!( + "eth: got eth_sub_result but no matching sub {} found, unsubscribing", + sub_id + ), ) .await; kernel_message( - &state.our.clone(), + &state.our, km.id, - km.source.clone(), + km.source, None, true, None, @@ -394,6 +410,7 @@ async fn handle_message( &state.send_to_loop, ) .await; + Ok(()) } IncomingReq::SubKeepalive(sub_id) => { // source expects that we have a local sub for them with this id @@ -426,11 +443,11 @@ async fn handle_message( &state.send_to_loop, ) .await; + Ok(()) } } } } - Ok(()) } async fn handle_eth_action( @@ -488,7 +505,10 @@ async fn handle_eth_action( let Some(mut sub_map) = state.active_subscriptions.get_mut(&km.source) else { verbose_print( &state.print_tx, - "eth: got unsubscribe but no matching subscription found", + &format!( + "eth: got unsubscribe from {} but no subscription found", + km.source + ), ) .await; error_message( @@ -522,7 +542,10 @@ async fn handle_eth_action( } else { verbose_print( &state.print_tx, - "eth: got unsubscribe but no matching subscription found", + &format!( + "eth: got unsubscribe from {} but no subscription {} found", + km.source, sub_id + ), ) .await; error_message( diff --git a/kinode/src/eth/subscription.rs b/kinode/src/eth/subscription.rs index 875de5038..21e78021c 100644 --- a/kinode/src/eth/subscription.rs +++ b/kinode/src/eth/subscription.rs @@ -500,7 +500,7 @@ async fn maintain_remote_subscription( None, true, None, - EthAction::UnsubscribeLogs(sub_id), + EthAction::UnsubscribeLogs(remote_sub_id), send_to_loop, ) .await;